def testNormalTransmitResetMessage(self):
ti = TransmitIntent('addr', 'msg')
self.assertEqual(ti.targetAddr, 'addr')
self.assertEqual(ti.message, 'msg')
ti.changeMessage('message2')
self.assertEqual(ti.targetAddr, 'addr')
self.assertEqual(ti.message, 'message2')
def testTransmitIntentSetResult(self):
ti = TransmitIntent('addr', 'msg')
assert None == ti.result
ti.result = SendStatus.Sent
assert ti.result == SendStatus.Sent
ti.result = SendStatus.Failed
assert ti.result == SendStatus.Failed
def testTransmitIntentRetryTimingExceedsLimit(self):
maxPeriod = timedelta(seconds=90)
period = timedelta(microseconds=1)
now = 1.23
timepad = timedelta(microseconds=10) # avoid float imprecision
with update_elapsed_time(now, timedelta(0)):
ti = TransmitIntent('addr', 'msg',
maxPeriod=maxPeriod,
retryPeriod=period)
assert not ti.timeToRetry()
timeoffset = timedelta(0)
for N in range(MAX_TRANSMIT_RETRIES+1):
# Indicate "failure" and the need to retry
with update_elapsed_time(now, timeoffset + timepad):
assert ti.retry()
# Wait for the indication that it is time to retry
time_to_retry = False
for x in range(90):
with update_elapsed_time(now, timeoffset + timepad):
# Only call timeToRetry once, because it auto-resets
time_to_retry = ti.timeToRetry()
if time_to_retry: break
timeoffset += (period + (period / 2))
# = period * 1.5, but python2 cannot multiply
# timedelta by fractions.
assert time_to_retry
with update_elapsed_time(now, timeoffset + timepad):
assert not ti.retry()
def testNormalTransmitResetMessage(self):
ti = TransmitIntent('addr', 'msg')
assert ti.targetAddr == 'addr'
assert ti.message == 'msg'
ti.changeMessage('message2')
assert ti.targetAddr == 'addr'
assert ti.message == 'message2'
def testTransmitIntentSetResult(self):
ti = TransmitIntent('addr', 'msg')
self.assertEqual(None, ti.result)
ti.result = SendStatus.Sent
self.assertEqual(ti.result, SendStatus.Sent)
ti.result = SendStatus.Failed
self.assertEqual(ti.result, SendStatus.Failed)
def testTransmitIntentDelay(self):
maxPeriod = timedelta(milliseconds=90)
period = timedelta(milliseconds=30)
ti = TransmitIntent('addr', 'msg', maxPeriod=maxPeriod, retryPeriod=period)
delay = ti.delay()
self.assertGreater(delay, timedelta(milliseconds=88))
self.assertLess(delay, timedelta(milliseconds=91))
def testTransmitIntentDelay(self):
maxPeriod = timedelta(milliseconds=90)
period = timedelta(milliseconds=30)
ti = TransmitIntent('addr', 'msg',
maxPeriod=maxPeriod,
retryPeriod=period)
delay = ti.delay()
assert delay > timedelta(milliseconds=88)
assert delay < timedelta(milliseconds=91)
def testTransmitIntentRetryTimingExceedsLimit(self):
maxPeriod = timedelta(seconds=90)
period = timedelta(microseconds=1)
ti = TransmitIntent('addr', 'msg', maxPeriod=maxPeriod, retryPeriod=period)
self.assertFalse(ti.timeToRetry())
for N in range(MAX_TRANSMIT_RETRIES+1):
# Indicate "failure" and the need to retry
self.assertTrue(ti.retry())
# Wait for the indication that it is time to retry
time_to_retry = False
for x in range(90):
# Only call timeToRetry once, because it auto-resets
time_to_retry = ti.timeToRetry()
if time_to_retry: break
sleep(timePeriodSeconds(period) * 1.5)
self.assertTrue(time_to_retry)
self.assertFalse(ti.retry())
def testTransmitIntentCallbackFailureFailed(self):
ti = TransmitIntent('addr', 'msg')
ti.result = SendStatus.Failed
# Ensure no exception thrown
ti.completionCallback()
# And again
ti.completionCallback()
def testTransmitIntentCallbackFailureFailedWithChangedTargetsAdded(self):
self.successes = []
self.failures = []
ti = TransmitIntent('addr', 'msg', onSuccess = self._success, onError = self._failed)
ti.result = SendStatus.Failed
# Ensure no exception thrown
ti.completionCallback()
self.assertEqual(self.successes, [])
self.assertEqual(self.failures, [(SendStatus.Failed, ti)])
# And again
ti.addCallback(self._success, self._failed)
ti.completionCallback()
self.assertEqual(self.successes, [])
self.assertEqual(self.failures, [(SendStatus.Failed, ti), (SendStatus.Failed, ti)])
def _sendPendingActorResponse(self, requestEnvelope, actualAddress,
errorCode=None, errorStr=None):
# actualAddress is None for failure
if actualAddress is None and errorCode is None:
raise ValueError('Must specify either actualAddress or errorCode')
self._send_intent(
TransmitIntent(
requestEnvelope.message.forActor or requestEnvelope.sender,
PendingActorResponse(requestEnvelope.message.forActor,
requestEnvelope.message.instanceNum,
requestEnvelope.message.globalName,
errorCode=errorCode,
errorStr=errorStr,
actualAddress=actualAddress)))
def testTwoSendsIntentTimeoutIgnored(self):
self.sends = []
hs = HysteresisDelaySender(self.send,
hysteresis_min_period = timedelta(milliseconds=100),
hysteresis_max_period = timedelta(milliseconds=110),
hysteresis_rate = 2)
intent1 = TransmitIntent('addr1', 'msg1')
intent2 = TransmitIntent('addr1', 'msg2', maxPeriod=timedelta(milliseconds=10))
hs.sendWithHysteresis(intent1)
hs.sendWithHysteresis(intent2)
# First was sent immediately, second is delayed
assert 1 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=110) >= hs.delay.remaining()
assert timedelta(milliseconds=95) < hs.delay.remaining()
print('Remaining seconds: %s (%s)'%(hs.delay.remainingSeconds(),
type(hs.delay.remainingSeconds())))
sleep(hs.delay.remainingSeconds())
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert intent2 == self.sends[1]
def _pendingActorReady(self, childInstance, actualAddress):
if childInstance in self._pendingChildren:
gName = self._pendingChildren[childInstance].message.globalName
if gName:
if gName in self._globalNames:
# This is the loser of the race... just kill it.
self._send_intent(
TransmitIntent(actualAddress,
ActorExitRequest(recursive=True)))
actualAddress = self._globalNames[gName]
else:
self._globalNames[gName] = actualAddress
return super(GlobalNamesAdmin,
self)._pendingActorReady(childInstance, actualAddress)
def testTransmitIntentCallbackSuccessWithChangedTargetsAdded(self):
self.successes = []
self.failures = []
ti = TransmitIntent('addr', 'msg',
onSuccess = self._success,
onError = self._failed)
ti.result = SendStatus.Sent
# Ensure no exception thrown
ti.completionCallback()
assert self.successes == [(SendStatus.Sent, ti)]
assert self.failures == []
# And again
ti.addCallback(self._success, self._failed)
ti.completionCallback()
assert self.successes == [(SendStatus.Sent, ti), (SendStatus.Sent, ti)]
assert self.failures == []
def handle(self, record):
# Can't pickle traceback objects, so pre-format them. Sorry,
# more logging internals here.
# Originally *all* messages were forwarded from actors to the
# logging actor, and the filtering/level management was
# performed by the latter.
#
# However, some packages
# (e.g. sqlalchemy) generate large amounts of debug logging
# with the expectation that those are normally filtered and
# hidden. This could cause excessive amounts of transmits
# from an actor to the logger, to the point that the TxOnly
# threshold was exceeded and actor responsiveness was
# degraded.
#
# To avoid that degradation, the level set for this handler is
# used to filter messages prior to sending them to the logger.
# The downside is that the logging level cannot be globally
# changed, although it can be adjusted on a per-actor basis.
if record.levelno < self.level:
return
global logTransport
if record.exc_info:
if not record.exc_text:
excinfo = traceback.format_exception(record.exc_info[0],
record.exc_info[1],
record.exc_info[2])
record.exc_text = '\n'.join(excinfo)
record.exc_info = None
record.__dict__['actorAddress'] = str(logTransport.myAddress)
try:
msg = record.getMessage()
except Exception:
newargs = []
for arg in record.args:
try:
newargs.append(str(arg))
except Exception:
newargs.append('<un-str-able argument>')
record.args = newargs
try:
msg = record.getMessage()
except Exception:
msg = '<un-str-able message>'
record.msg = msg
record.args = None
logTransport.scheduleTransmit(None,
TransmitIntent(self._fwdAddr, record))
def unloadActorSource(self, sourceHash):
if sourceHash in self._sources:
msg = UnloadedSource(self._sources[sourceHash].srcHash,
self._sources[sourceHash].srcInfo)
for each in self._sourceNotifications:
self._send_intent(TransmitIntent(each, msg))
del self._sources[sourceHash]
for pnum, metapath in enumerate(sys.meta_path):
if getattr(metapath, 'srcHash', None) == sourceHash:
rmmods = [M for M in sys.modules
if M and M.startswith(metapath.hashRoot())]
for each in rmmods:
del sys.modules[each]
del sys.meta_path[pnum]
break
def h_NotifyOnSourceAvailability(self, envelope):
address = envelope.message.notificationAddress
enable = envelope.message.enable
all_except = [A for A in self._sourceNotifications if A != address]
if enable:
self._sourceNotifications = all_except + [address]
for each in self._sources:
if self._sources[each].source_valid:
self._send_intent(
TransmitIntent(
address,
LoadedSource(self._sources[each].srcHash,
self._sources[each].srcInfo)))
else:
self._sourceNotifications = all_except
def handle(self, record):
# Can't pickle traceback objects, so pre-format them. Sorry,
# more logging internals here.
if record.exc_info:
if not record.exc_text:
excinfo = traceback.format_exception(record.exc_info[0],
record.exc_info[1],
record.exc_info[2])
record.exc_text = '\n'.join(excinfo)
record.exc_info = None
record.__dict__['actorAddress'] = str(self._transport.myAddress)
msg = record.getMessage()
record.msg = msg
record.args = None
self._transport.scheduleTransmit(None,
TransmitIntent(self._fwdAddr, record))
def h_NotifyOnSourceAvailability(self, envelope):
address = envelope.message.notificationAddress
enable = envelope.message.enable
all_except = list(
filter(lambda a: a != address, self._sourceNotifications))
if enable:
self._sourceNotifications = all_except + [address]
for each in self._sources:
if hasattr(self._sources[each], 'srcHash'):
self._send_intent(
TransmitIntent(
address,
LoadedSource(self._sources[each].srcHash,
self._sources[each].srcInfo)))
else:
self._sourceNotifications = all_except
def _performIO(self, iolist):
for msg in iolist:
if isinstance(msg, HysteresisCancel):
self._hysteresisSender.cancelSends(msg.cancel_addr)
elif isinstance(msg, HysteresisSend):
#self._send_intent(msg)
self._hysteresisSender.sendWithHysteresis(msg)
elif isinstance(msg, LogAggregator):
if getattr(self, 'asLogger', None):
thesplog('Setting log aggregator of %s to %s', self.asLogger, msg.aggregatorAddress)
self._send_intent(TransmitIntent(self.asLogger, msg))
elif isinstance(msg, LostRemote):
if hasattr(self.transport, 'lostRemote'):
self.transport.lostRemote(msg.lost_addr)
else:
self._send_intent(msg)
def h_NotifyOnSystemRegistration(self, envelope):
if envelope.message.enableNotification:
newRegistrant = envelope.sender not in self._conventionNotificationHandlers
if newRegistrant:
self._conventionNotificationHandlers.add(envelope.sender)
# Now update the registrant on the current state of all convention members
for member in self._conventionMembers:
self._send_intent(
TransmitIntent(
envelope.sender,
ActorSystemConventionUpdate(
member, self._conventionMembers[member].
remoteCapabilities, True)))
else:
self._conventionNotificationHandlers.discard(envelope.sender)
return True
def _loadValidatedActorSource(self, sourceHash, sourceZip, sourceInfo):
# Validate the source file; this doesn't actually utilize the
# sourceZip, but it ensures that the sourceZip isn't garbage
# before registering it as active source.
if sourceHash not in self._sources:
logging.getLogger('Thespian').warning(
'Provided validated source with no or expired request'
', hash %s; ignoring.', sourceHash)
return
try:
f = SourceHashFinder(sourceHash, lambda v: v, sourceZip)
namelist = f.getZipNames()
logging.getLogger('Thespian').info(
'Validated source hash %s - %s, %s modules (%s)',
sourceHash, sourceInfo, len(namelist),
', '.join(namelist if len(namelist) < 10 else
namelist[:9] + ['...']))
except Exception as ex:
logging.getLogger('Thespian')\
.error('Validated source (hash %s) is corrupted: %s',
sourceHash, ex)
return
if self._sources[sourceHash].source_valid:
# If a duplicate source load request is made while the
# first is still being validated by the Source Authority,
# another request will be sent to the Source Authority and
# the latter response will be a duplicate here and can
# simply be dropped.
return
# Store this registered source
pending_actors = self._sources[sourceHash].pending_actors
self._sources[sourceHash] = ValidSource(sourceHash,
self._sources[sourceHash].orig_data,
sourceZip,
str(sourceInfo))
for each in pending_actors:
self.h_PendingActor(each)
msg = LoadedSource(self._sources[sourceHash].srcHash,
self._sources[sourceHash].srcInfo)
for each in self._sourceNotifications:
self._send_intent(TransmitIntent(each, msg))
def got_convention_deregister(self, deregmsg):
self._sCBStats.inc('Admin Handle Convention De-registration')
remoteAdmin = deregmsg.adminAddress
if remoteAdmin == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog('Convention deregistration from %s is an invalid address; ignoring.',
remoteAdmin,
level=logging.WARNING)
rmsgs = []
if getattr(deregmsg, 'preRegistered', False): # see definition for getattr use
existing = self._conventionMembers.find(remoteAdmin)
if existing:
existing.preRegistered = None
rmsgs.append(TransmitIntent(remoteAdmin, ConventionDeRegister(self.myAddress)))
return rmsgs + self._remote_system_cleanup(remoteAdmin)
def testTransmitIntentCallbackFailureFailedWithTarget(self):
self.successes = []
self.failures = []
ti = TransmitIntent('addr', 'msg',
onSuccess = self._success,
onError = self._failed)
ti.result = SendStatus.Failed
# Ensure no exception thrown
ti.completionCallback()
assert self.successes == []
assert self.failures == [(SendStatus.Failed, ti)]
# And again
ti.completionCallback()
assert self.successes == []
assert self.failures == [(SendStatus.Failed, ti)]
def got_system_shutdown(self):
gen_ops = lambda addr: [
HysteresisCancel(addr),
TransmitIntent(addr, ConventionDeRegister(self.myAddress)),
]
if self.conventionLeaderAddr and \
self.conventionLeaderAddr != self.myAddress:
thesplog('Admin de-registering with Convention @ %s',
str(self.conventionLeaderAddr),
level=logging.INFO,
primary=True)
return gen_ops(self.conventionLeaderAddr)
return join(
fmap(gen_ops, [
M.remoteAddress for M in self._conventionMembers.values()
if M.remoteAddress != self.myAddress
]))
def forward_pending_to_remote_system(self, childClass, envelope,
sourceHash, acceptsCaps):
alreadyTried = getattr(envelope.message, 'alreadyTried', [])
ct = currentTime()
if self.myAddress not in alreadyTried:
# Don't send request back to this actor system: it cannot
# handle it
alreadyTried.append(self.myAddress)
remoteCandidates = [
K for K in self._conventionMembers.values()
if not K.registryValid.view(ct).expired()
and K.remoteAddress != envelope.sender # source Admin
and K.remoteAddress not in alreadyTried
and acceptsCaps(K.remoteCapabilities)
]
if not remoteCandidates:
if self.isConventionLeader() or not self.conventionLeaderAddr:
raise NoCompatibleSystemForActor(
childClass, 'No known ActorSystems can handle a %s for %s',
childClass, envelope.message.forActor)
# Let the Convention Leader try to find an appropriate ActorSystem
bestC = self.conventionLeaderAddr
else:
# distribute equally amongst candidates
C = [(K.remoteAddress, len(K.hasRemoteActors))
for K in remoteCandidates]
bestC = foldl(
lambda best, possible: best
if best[1] <= possible[1] else possible, C)[0]
thesplog('Requesting creation of %s%s on remote admin %s',
envelope.message.actorClassName,
' (%s)' % sourceHash if sourceHash else '', bestC)
if bestC in alreadyTried:
return [] # Have to give up, no-one can handle this
# Don't send request to this remote again, it has already
# been tried. This would also be indicated by that system
# performing the add of self.myAddress as below, but if
# there is disagreement between the local and remote
# addresses, this addition will prevent continual
# bounceback.
alreadyTried.append(bestC)
envelope.message.alreadyTried = alreadyTried
return [TransmitIntent(bestC, envelope.message)]
def test_pendingCallbacksClearQueueAndMoreRunsRunAdditionalQueuedOnMoreCompletions(
self):
numExtras = len(self.extraTransmitIds)
self.test_extraPendingCallbackCompletionsDoNothing()
# Add more transmits to reach MAX_PENDING_TRANSMITS again
for _moreExtras in range(numExtras + 3):
self.testTrans.forTestingCompleteAPendingIntent(SendStatus.Sent)
expectedCBCount = 2 * (numExtras + 3)
self.assertEqual(self.successCBcalls, expectedCBCount)
self.assertEqual(MAX_PENDING_TRANSMITS + numExtras,
len(self.testTrans.intents))
self.assertEqual(
numExtras,
len([
I for I in self.testTrans.intents
if I.message in self.extraTransmitIds
]))
# Now send more and make sure they are queued
for count in self.extraTransmitIds:
self.testTrans.scheduleTransmit(
None,
TransmitIntent(ActorAddress(3.5), count, self.successCB,
self.failureCB))
self.assertEqual(self.successCBcalls, expectedCBCount)
self.assertEqual(MAX_PENDING_TRANSMITS + 2 * numExtras,
len(self.testTrans.intents))
self.assertEqual(
2 * numExtras,
len([
I for I in self.testTrans.intents
if I.message in self.extraTransmitIds
]))
# And verify that more completions cause the newly Queued to be run
for _extras in range(numExtras):
self.testTrans.forTestingCompleteAPendingIntent(SendStatus.Sent)
expectedCBCount += numExtras
self.assertEqual(self.successCBcalls, expectedCBCount)
self.assertEqual(MAX_PENDING_TRANSMITS + numExtras + numExtras,
len(self.testTrans.intents))
self.assertEqual(
2 * numExtras,
len([
I for I in self.testTrans.intents
if I.message in self.extraTransmitIds
]))
def testTransmitIntentCallbackFailureNotSentWithTarget(self):
self.successes = []
self.failures = []
ti = TransmitIntent('addr',
'msg',
onSuccess=self._success,
onError=self._failed)
ti.result = SendStatus.NotSent
# Ensure no exception thrown
ti.completionCallback()
self.assertEqual(self.successes, [])
self.assertEqual(self.failures, [(SendStatus.NotSent, ti)])
# And again
ti.completionCallback()
self.assertEqual(self.successes, [])
self.assertEqual(self.failures, [(SendStatus.NotSent, ti)])
def h_PendingActor(self, envelope):
gName = envelope.message.globalName
if not gName:
return super(GlobalNamesAdmin, self).h_PendingActor(envelope)
if gName in self._globalNames:
# Actor already registered with this name... here it is.
self._send_intent(
TransmitIntent(
envelope.sender,
PendingActorResponse(
envelope.message.forActor,
envelope.message.instanceNum,
gName,
actualAddress=self._globalNames[gName])))
return True
return super(GlobalNamesAdmin, self).h_PendingActor(envelope)
def testTransmitIntentCallbackFailureNotSentWithTarget(self):
self.successes = []
self.failures = []
ti = TransmitIntent('addr', 'msg', onSuccess = self._success, onError = self._failed)
ti.result = SendStatus.NotSent
# Ensure no exception thrown
ti.completionCallback()
self.assertEqual(self.successes, [])
self.assertEqual(self.failures, [(SendStatus.NotSent, ti)])
# And again
ti.completionCallback()
self.assertEqual(self.successes, [])
self.assertEqual(self.failures, [(SendStatus.NotSent, ti)])
def testTransmitIntentRetryTimingExceedsLimit(self):
maxPeriod = timedelta(seconds=90)
period = timedelta(microseconds=1)
ti = TransmitIntent('addr',
'msg',
maxPeriod=maxPeriod,
retryPeriod=period)
self.assertFalse(ti.timeToRetry())
for N in range(MAX_TRANSMIT_RETRIES + 1):
self.assertTrue(ti.retry())
for x in range(90):
if ti.timeToRetry(): break
sleep(timePeriodSeconds(period))
self.assertTrue(ti.timeToRetry())
self.assertFalse(ti.retry())
def run(self):
try:
while True:
r = self.transport.run(self.handleIncoming, None)
if isinstance(r, Thespian__UpdateWork):
# tickle the transmit queues
self._send_intent(TransmitIntent(self.myAddress, r))
continue
# Expects that on completion of self.transport.run
# that the Actor is done processing and that it has
# been shutdown gracefully.
break
except Exception:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def testTransmitIntentRetryTimingExceedsLimit(self):
maxPeriod = timedelta(seconds=90)
period = timedelta(microseconds=1)
ti = TransmitIntent('addr', 'msg', maxPeriod=maxPeriod, retryPeriod=period)
self.assertFalse(ti.timeToRetry())
for N in range(MAX_TRANSMIT_RETRIES+1):
self.assertTrue(ti.retry())
for x in range(90):
if ti.timeToRetry(): break
sleep(timePeriodSeconds(period))
self.assertTrue(ti.timeToRetry())
self.assertFalse(ti.retry())
def _get_missing_source_for_hash(self, sourceHash, createActorEnvelope):
# If this request was forwarded by a remote Admin and the
# sourceHash is not known locally, request it from the sending
# remote Admin
if self._cstate.sentByRemoteAdmin(createActorEnvelope) and \
self._acceptsRemoteLoadedSourcesFrom(createActorEnvelope):
self._sources[sourceHash] = PendingSource(sourceHash, None)
self._sources[sourceHash].pending_actors.append(
createActorEnvelope)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(
createActorEnvelope.sender,
SourceHashTransferRequest(sourceHash,
bool(self._sourceAuthority))))
# sent with hysteresis, so break out to local _run
return False
# No remote Admin to send the source, so fail as normal.
return super(ConventioneerAdmin, self)._get_missing_source_for_hash(
sourceHash, createActorEnvelope)
def exit_convention(self):
self.invited = False
gen_ops = lambda addr: [HysteresisCancel(addr),
TransmitIntent(addr,
ConventionDeRegister(self.myAddress)),
]
terminate = lambda a: [ self._remote_system_cleanup(a), gen_ops(a) ][-1]
if self.conventionLeaderAddr and \
self.conventionLeaderAddr != self.myAddress:
thesplog('Admin de-registering with Convention @ %s',
str(self.conventionLeaderAddr),
level=logging.INFO, primary=True)
# Cache convention leader address because it might get reset by terminate()
claddr = self.conventionLeaderAddr
terminate(self.conventionLeaderAddr)
return gen_ops(claddr)
return join(fmap(terminate,
[M.remoteAddress
for M in self._conventionMembers.values()
if M.remoteAddress != self.myAddress]))
def testTransmitIntentCallbackFailureFailedWithTarget(self):
self.successes = []
self.failures = []
ti = TransmitIntent('addr', 'msg',
onSuccess = self._success,
onError = self._failed)
ti.result = SendStatus.Failed
# Ensure no exception thrown
ti.completionCallback()
assert self.successes == []
assert self.failures == [(SendStatus.Failed, ti)]
# And again
ti.completionCallback()
assert self.successes == []
assert self.failures == [(SendStatus.Failed, ti)]
def h_ValidateSource(self, envelope):
sourceHash = envelope.message.sourceHash
if not envelope.message.sourceData:
self.unloadActorSource(sourceHash)
logging.getLogger('Thespian')\
.info('Source hash %s unloaded', sourceHash)
return
if sourceHash in self._sources:
logging.getLogger('Thespian')\
.info('Source hash %s (%s) already loaded', sourceHash,
self._sources[sourceHash].srcInfo
if isinstance(self._sources[sourceHash], ValidSource)
else '<pending>')
return
if self._sourceAuthority:
self._send_intent(
TransmitIntent(self._sourceAuthority, envelope.message))
return
# Any attempt to load sources is ignored if there is no active
# Source Authority. This is a security measure to protect the
# un-protected.
logging.getLogger('Thespian').warning(
'No source authority to validate source hash %s', sourceHash)
def _checkConvention(self):
if self.isConventionLeader():
missing = []
for each in self._conventionMembers:
if self._conventionMembers[each].registryValid.expired():
missing.append(each)
for each in missing:
thesplog('%s missed %d checkins (%s); assuming it has died',
str(self._conventionMembers[each]),
CONVENTION_REGISTRATION_MISS_MAX,
str(self._conventionMembers[each].registryValid),
level=logging.WARNING,
primary=True)
self._remoteSystemCleanup(
self._conventionMembers[each].remoteAddress)
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
else:
# Re-register with the Convention if it's time
if self._conventionAddress and self._conventionRegistration.expired(
):
self.setupConvention()
for each in self._conventionMembers:
member = self._conventionMembers[each]
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
member.preRegistered.pingValid = ExpiryTime(
CONVENTION_RESTART_PERIOD if member.registryValid.expired(
) else CONVENTION_REREGISTRATION_PERIOD)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(member.remoteAddress,
ConventionInvite(),
onSuccess=self._preRegQueryNotPending,
onError=self._preRegQueryNotPending))
def startupASLogger(addrOfStarter, logEndpoint, logDefs, transportClass,
aggregatorAddress):
# Dirty trick here to completely re-initialize logging in this
# process... something the standard Python logging interface does
# not allow via the API. We also do not want to run
# logging.shutdown() because (a) that does not do enough to reset,
# and (b) it shuts down handlers, but we want to leave the
# parent's handlers alone. Dirty trick here to completely
# re-initialize logging in this process... something the standard
# Python logging interface does not allow via the API.
logging.root = logging.RootLogger(logging.WARNING)
logging.Logger.root = logging.root
logging.Logger.manager = logging.Manager(logging.Logger.root)
if logDefs:
dictConfig(logDefs)
else:
logging.basicConfig()
# Disable thesplog from within the logging process (by setting the
# logfile size to zero) to try to avoid recursive logging loops.
thesplog_control(logging.WARNING, False, 0)
#logging.info('ActorSystem Logging Initialized')
transport = transportClass(logEndpoint)
setProcName('logger', transport.myAddress)
transport.scheduleTransmit(
None, TransmitIntent(addrOfStarter, LoggerConnected()))
fdup = None
last_exception_time = None
exception_count = 0
while True:
try:
r = transport.run(None)
if isinstance(r, Thespian__UpdateWork):
transport.scheduleTransmit(
TransmitIntent(transport.myAddress, r))
continue
logrecord = r.message
if isinstance(logrecord, LoggerExitRequest):
logging.info('ActorSystem Logging Shutdown')
return
elif isinstance(logrecord, LoggerFileDup):
fdup = getattr(logrecord, 'fname', None)
elif isinstance(logrecord, LogAggregator):
aggregatorAddress = logrecord.aggregatorAddress
elif isinstance(logrecord, logging.LogRecord):
logging.getLogger(logrecord.name).handle(logrecord)
if fdup:
with open(fdup, 'a') as ldf:
ldf.write('%s\n' % str(logrecord))
if aggregatorAddress and \
logrecord.levelno >= logging.WARNING:
transport.scheduleTransmit(
None, TransmitIntent(aggregatorAddress, logrecord))
else:
logging.warn('Unknown message rcvd by logger: %s' %
str(logrecord))
except Exception as ex:
thesplog('Thespian Logger aborting (#%d) with error %s',
exception_count,
ex,
exc_info=True)
if last_exception_time is None or \
last_exception_time.view().expired():
last_exception_time = ExpirationTimer(timedelta(seconds=1))
exception_count = 0
else:
exception_count += 1
if exception_count >= MAX_LOGGING_EXCEPTIONS_PER_SECOND:
thesplog(
'Too many Thespian Logger exceptions (#%d in %s); exiting!',
exception_count,
timedelta(seconds=1) -
last_exception_time.view().remaining())
return
def testTransmitIntentRetry(self):
ti = TransmitIntent('addr', 'msg')
for x in range(MAX_TRANSMIT_RETRIES+1):
assert ti.retry()
assert not ti.retry()
def testTransmitIntentRetryTiming(self):
maxPeriod = timedelta(milliseconds=90)
period = timedelta(milliseconds=30)
now = 0.01
timepad = timedelta(microseconds=10) # avoid float imprecision
with update_elapsed_time(now, timedelta(0)):
ti = TransmitIntent('addr', 'msg',
maxPeriod=maxPeriod,
retryPeriod=period)
assert not ti.timeToRetry()
with update_elapsed_time(now, period + timepad):
assert not ti.timeToRetry()
assert ti.retry()
assert not ti.timeToRetry()
with update_elapsed_time(now, period + period + timepad):
assert ti.timeToRetry()
assert ti.retry()
assert not ti.timeToRetry()
with update_elapsed_time(now, period * 3 + timepad):
assert not ti.timeToRetry() # Each retry increases
with update_elapsed_time(now, period * 4 + timepad):
assert ti.timeToRetry()
assert not ti.retry() # Exceeds maximum time
def testNormalTransmitIdentification(self):
ti = TransmitIntent('addr', 'msg')
# Just ensure no exceptions are thrown
self.assertTrue(ti.identify())
def testTransmitIntentRetryTiming(self):
maxPeriod = timedelta(milliseconds=90)
period = timedelta(milliseconds=30)
ti = TransmitIntent('addr', 'msg', maxPeriod=maxPeriod, retryPeriod=period)
self.assertFalse(ti.timeToRetry())
sleep(timePeriodSeconds(period))
self.assertFalse(ti.timeToRetry())
self.assertTrue(ti.retry())
self.assertFalse(ti.timeToRetry())
sleep(timePeriodSeconds(period))
self.assertTrue(ti.timeToRetry())
self.assertTrue(ti.retry())
self.assertFalse(ti.timeToRetry())
sleep(timePeriodSeconds(period))
self.assertFalse(ti.timeToRetry()) # Each retry increases
sleep(timePeriodSeconds(period))
self.assertTrue(ti.timeToRetry())
self.assertFalse(ti.retry()) # Exceeds maximum time
def got_convention_register(self, regmsg):
# Called when remote convention member has sent a
# ConventionRegister message. This is first called the leader
# when the member registers with the leader, and then on the
# member when the leader responds with same. Thus the current
# node could be a member, a potential leader, the current
# leader, or a potential leader with higher potential than the
# current leader and which should become the new leader.
self._sCBStats.inc('Admin Handle Convention Registration')
if self._invited and not self.conventionLeaderAddr:
# Lost connection to an invitation-only convention.
# Cannot join again until another invitation is received.
return []
# Remote member may re-register if changing capabilities
rmsgs = []
registrant = regmsg.adminAddress
prereg = getattr(regmsg, 'preRegister',
False) # getattr used; see definition
existing = self._conventionMembers.find(registrant)
thesplog('Got Convention %sregistration from %s (%s) (new? %s)',
'pre-' if prereg else '',
registrant,
'first time' if regmsg.firstTime else 're-registering',
not existing,
level=logging.DEBUG)
if registrant == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog(
'Convention registration from %s is an invalid address; ignoring.',
registrant,
level=logging.WARNING)
return rmsgs
existingPreReg = (
# existing.preRegOnly
# or existing.preRegistered
existing.permanentEntry) if existing else False
notify = (not existing or existing.preRegOnly) and not prereg
if regmsg.firstTime or not existing:
if existing:
existing = None
notify = not prereg
rmsgs.extend(self._remote_system_cleanup(registrant))
newmember = ConventionMemberData(registrant, regmsg.capabilities,
prereg)
if prereg or existingPreReg:
newmember.preRegistered = PreRegistration()
self._conventionMembers.add(registrant, newmember)
else:
existing.refresh(regmsg.capabilities, prereg or existingPreReg)
if not prereg:
existing.preRegOnly = False
if not self.isConventionLeader():
self._conventionRegistration = ExpirationTimer(
CONVENTION_REREGISTRATION_PERIOD)
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
# Convention Members normally periodically initiate a
# membership message, to which the leader confirms by
# responding.
#if self.isConventionLeader() or prereg or regmsg.firstTime:
if prereg:
# If this was a pre-registration, that identifies this
# system as the "leader" for that remote. Also, if the
# remote sent this because it was a pre-registration
# leader, it doesn't yet have all the member information
# so the member should respond.
rmsgs.append(HysteresisCancel(registrant))
rmsgs.append(TransmitIntent(registrant, ConventionInvite()))
elif (self.isConventionLeader() or prereg or regmsg.firstTime or \
(existing and existing.permanentEntry)):
# If we are the Convention Leader, this would be the point to
# inform all other registrants of the new registrant. At
# present, there is no reciprocity here, so just update the
# new registrant with the leader's info.
rmsgs.append(
TransmitIntent(
registrant,
ConventionRegister(self.myAddress, self.capabilities)))
if notify:
rmsgs.extend(
self._notifications_of(
ActorSystemConventionUpdate(registrant,
regmsg.capabilities, True)))
return rmsgs
def testTransmitIntentRetry(self):
ti = TransmitIntent('addr', 'msg')
for x in range(MAX_TRANSMIT_RETRIES+1):
self.assertTrue(ti.retry())
self.assertFalse(ti.retry())
def _notifications_of(self, msg):
return [
TransmitIntent(H, msg)
for H in self._conventionNotificationHandlers
]
def _remote_system_cleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership. This is also called on
a First Time connection from the remote to discard any
previous connection information.
"""
thesplog('Convention cleanup or deregistration for %s (known? %s)',
registrant,
bool(self._conventionMembers.find(registrant)),
level=logging.INFO)
rmsgs = [LostRemote(registrant)]
cmr = self._conventionMembers.find(registrant)
if not cmr or cmr.preRegOnly:
return []
# Send exited notification to conventionNotificationHandler (if any)
for each in self._conventionNotificationHandlers:
rmsgs.append(
TransmitIntent(
each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
cla = self.conventionLeaderAddr
self._conventionMembers.rmv(registrant)
if registrant == cla:
if self._invited:
# Don't clear invited: once invited, that
# perpetually indicates this should be only a
# member and never a leader.
self._conventionAddress = [None]
else:
rmsgs.extend(self.setup_convention())
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpirationTimer(None)
cmr.preRegOnly = True
return rmsgs + [HysteresisCancel(registrant)]
def test_sendIntentToTransport(self):
testTrans = FakeTransport()
testIntent = TransmitIntent(ActorAddress(None), 'message',
self.successCB, self.failureCB)
testTrans.scheduleTransmit(None, testIntent)
assert 1 == len(testTrans.intents)
def testTwentySendsSameAddressSameMessageTypeSendAfterDelay(self):
self.sends = []
min_h_period = timedelta(milliseconds=2)
max_h_period = timedelta(milliseconds=20)
h_rate = 2
hs = HysteresisDelaySender(self.send,
hysteresis_min_period=min_h_period,
hysteresis_max_period=max_h_period,
hysteresis_rate=h_rate)
# Create the messages to send
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(20):
intents.append(TransmitIntent('addr1', 'msg'))
# Send all intents in rapid succession
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
# The hysteresis delay should be maxed out
t1 = hs.delay.remaining()
assert timedelta(seconds=0) != t1
assert max_h_period >= t1
# Wait the delay period and then check, which should send the
# (latest) queued messages. Because all the messages and
# target addresses are identical, this will actually only send
# a single message.
sleep(hs.delay.remainingSeconds())
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
# Verify that although the queued messages were sent, the
# hysteresis delay is not yet back to zero and additional
# sends are still blocked.
assert not hs.delay.expired() # refreshed and reduced in checkSends
hs.sendWithHysteresis(intents[0])
t1 = hs.delay.remaining() # send attempt probably bumped this up again
assert 2 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
# Verify that the hysteresis delay keeps dropping and
# eventually gets back to zero. After a drop, any pending
# sends that were blocked should be sent.
nsent = 2 # after first wait, checkSends will send the one just queued...
for x in range(100): # don't loop forever
if hs.delay.expired(): break
t2 = hs.delay.remaining()
assert timedelta(seconds=0) != t2
assert (x, t2) < (x, t1)
assert nsent == len(getattr(self, 'sends', []))
sleep(hs.delay.remainingSeconds())
t1 = t2
hs.checkSends()
if nsent == 2:
# All queued sends should now have been sent, but
# since they are all the same, there was only one more
# actual send.
nsent = 3
# Now verify hysteresis sender is back to the original state
assert 3 == len(getattr(self, 'sends', []))
hs.sendWithHysteresis(intents[1])
assert 4 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
assert intents[0] == self.sends[2]
assert intents[1] == self.sends[3]
# Verify that all intents got completed even though some were
# duplicates and not actually sent.
for each in intents:
assert each.result == SendStatus.Sent
