def testTwentySendsSameAddressSameMessageTypeSuccess(self):
self.sends = []
self.fails = []
hs = HysteresisDelaySender(self.send,
hysteresis_min_period = timedelta(milliseconds=2),
hysteresis_max_period = timedelta(milliseconds=10),
hysteresis_rate = 2)
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(20):
intents.append(TransmitIntent('addr1', 'msg',
onSuccess = self.successfulIntent,
onError = self.failedIntent))
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=10) >= hs.delay.remaining()
assert timedelta(milliseconds=9) < hs.delay.remaining()
print('Remaining seconds: %s (%s)'%(hs.delay.remainingSeconds(),
type(hs.delay.remainingSeconds())))
sleep(hs.delay.remainingSeconds())
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
assert 20 == len(getattr(self, 'successes', []))
assert 0 == len(getattr(self, 'fails', []))
def testEighteenSendsSameAddressThreeDifferentMessageTypes(self):
self.sends = []
hs = HysteresisDelaySender(self.send,
hysteresis_min_period = timedelta(milliseconds=2),
hysteresis_max_period = timedelta(milliseconds=10),
hysteresis_rate = 2)
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(18):
if num % 3 == 0: msg = 'msg'
elif num % 3 == 1: msg = 1
else: msg = True
intents.append(TransmitIntent('addr1', msg))
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=10) >= hs.delay.remaining()
assert timedelta(milliseconds=9) < hs.delay.remaining()
print('Remaining seconds: %s (%s)'%(hs.delay.remainingSeconds(),
type(hs.delay.remainingSeconds())))
sleep(hs.delay.remainingSeconds())
hs.checkSends()
# Only should have first message and last of each type
print('sends: %s' % [str(S.message) for S in self.sends])
assert 3 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-3] == self.sends[1]
assert intents[-1] == self.sends[2]
def testTwentySendsDifferentAddresses(self):
self.sends = []
hs = HysteresisDelaySender(self.send,
hysteresis_min_period = timedelta(milliseconds=2),
hysteresis_max_period = timedelta(milliseconds=10),
hysteresis_rate = 2)
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(20):
intents.append(TransmitIntent('addr%d'%num, 'msg'))
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=10) >= hs.delay.remaining()
assert timedelta(milliseconds=9) < hs.delay.remaining()
delayTime = hs.delay.remainingSeconds()
print('Remaining seconds: %s (%s)'%(delayTime, type(delayTime)))
sleep(delayTime)
hs.checkSends()
assert len(intents) == len(getattr(self, 'sends', []))
for num in range(len(intents)):
assert intents[num] == self.sends[num]
# Ensure that there are no more send attempts
sleep(hs.delay.remainingSeconds())
assert len(intents) == len(getattr(self, 'sends', []))
sleep(delayTime)
assert len(intents) == len(getattr(self, 'sends', []))
def testTwoSends(self):
self.sends = []
hs = HysteresisDelaySender(self.send,
hysteresis_min_period = timedelta(milliseconds=2),
hysteresis_max_period = timedelta(milliseconds=10),
hysteresis_rate = 2)
intent1 = TransmitIntent('addr1', 'msg1')
intent2 = TransmitIntent('addr1', 'msg2')
hs.sendWithHysteresis(intent1)
hs.sendWithHysteresis(intent2)
# First was sent immediately, second is delayed
assert 1 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=10) >= hs.delay.remaining()
delayTime = hs.delay.remainingSeconds()
print('Remaining seconds: %s (%s)'%(delayTime, type(delayTime)))
sleep(delayTime)
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert intent2 == self.sends[1]
# Ensure that there are no more send attempts
sleep(hs.delay.remainingSeconds())
assert 2 == len(getattr(self, 'sends', []))
sleep(delayTime)
assert 2 == len(getattr(self, 'sends', []))
def testSingleSend(self):
self.sends = []
hs = HysteresisDelaySender(self.send)
targetAddr = 'addr'
msg = 'msg'
intent = TransmitIntent(targetAddr, msg)
hs.sendWithHysteresis(intent)
# Should have been sent immediately
assert 1 == len(getattr(self, 'sends', []))
def __init__(self, *args, **kw):
super(ConventioneerAdmin, self).__init__(*args, **kw)
self._cstate = LocalConventionState(
self.myAddress,
self.capabilities,
self._sCBStats,
getattr(self.transport, 'getConventionAddress', lambda c: None))
self._hysteresisSender = HysteresisDelaySender(self._send_intent)
def testTwoSends(self):
self.sends = []
hs = HysteresisDelaySender(
self.send,
hysteresis_min_period=timedelta(milliseconds=2),
hysteresis_max_period=timedelta(milliseconds=10),
hysteresis_rate=2)
intent1 = TransmitIntent('addr1', 'msg1')
intent2 = TransmitIntent('addr1', 'msg2')
hs.sendWithHysteresis(intent1)
hs.sendWithHysteresis(intent2)
# First was sent immediately, second is delayed
assert 1 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=10) >= hs.delay.remaining()
delayTime = hs.delay.remainingSeconds()
print('Remaining seconds: %s (%s)' % (delayTime, type(delayTime)))
sleep(delayTime)
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert intent2 == self.sends[1]
# Ensure that there are no more send attempts
sleep(hs.delay.remainingSeconds())
assert 2 == len(getattr(self, 'sends', []))
sleep(delayTime)
assert 2 == len(getattr(self, 'sends', []))
def testEighteenSendsSameAddressThreeDifferentMessageTypes(self):
self.sends = []
hs = HysteresisDelaySender(
self.send,
hysteresis_min_period=timedelta(milliseconds=2),
hysteresis_max_period=timedelta(milliseconds=10),
hysteresis_rate=2)
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(18):
if num % 3 == 0: msg = 'msg'
elif num % 3 == 1: msg = 1
else: msg = True
intents.append(TransmitIntent('addr1', msg))
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=10) >= hs.delay.remaining()
assert timedelta(milliseconds=9) < hs.delay.remaining()
print('Remaining seconds: %s (%s)' %
(hs.delay.remainingSeconds(), type(hs.delay.remainingSeconds())))
sleep(hs.delay.remainingSeconds())
hs.checkSends()
# Only should have first message and last of each type
print('sends: %s' % [str(S.message) for S in self.sends])
assert 3 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-3] == self.sends[1]
assert intents[-1] == self.sends[2]
def testTwentySendsDifferentAddresses(self):
self.sends = []
hs = HysteresisDelaySender(
self.send,
hysteresis_min_period=timedelta(milliseconds=2),
hysteresis_max_period=timedelta(milliseconds=10),
hysteresis_rate=2)
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(20):
intents.append(TransmitIntent('addr%d' % num, 'msg'))
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=10) >= hs.delay.remaining()
assert timedelta(milliseconds=9) < hs.delay.remaining()
delayTime = hs.delay.remainingSeconds()
print('Remaining seconds: %s (%s)' % (delayTime, type(delayTime)))
sleep(delayTime)
hs.checkSends()
assert len(intents) == len(getattr(self, 'sends', []))
for num in range(len(intents)):
assert intents[num] == self.sends[num]
# Ensure that there are no more send attempts
sleep(hs.delay.remainingSeconds())
assert len(intents) == len(getattr(self, 'sends', []))
sleep(delayTime)
assert len(intents) == len(getattr(self, 'sends', []))
def testTwoSendsIntentTimeoutIgnored(self):
self.sends = []
hs = HysteresisDelaySender(
self.send,
hysteresis_min_period=timedelta(milliseconds=100),
hysteresis_max_period=timedelta(milliseconds=110),
hysteresis_rate=2)
intent1 = TransmitIntent('addr1', 'msg1')
intent2 = TransmitIntent('addr1',
'msg2',
maxPeriod=timedelta(milliseconds=10))
hs.sendWithHysteresis(intent1)
hs.sendWithHysteresis(intent2)
# First was sent immediately, second is delayed
assert 1 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=110) >= hs.delay.remaining()
assert timedelta(milliseconds=95) < hs.delay.remaining()
print('Remaining seconds: %s (%s)' %
(hs.delay.remainingSeconds(), type(hs.delay.remainingSeconds())))
sleep(hs.delay.remainingSeconds())
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert intent2 == self.sends[1]
def testTwentySendsSameAddressSameMessageTypeSuccess(self):
self.sends = []
self.fails = []
hs = HysteresisDelaySender(
self.send,
hysteresis_min_period=timedelta(milliseconds=2),
hysteresis_max_period=timedelta(milliseconds=10),
hysteresis_rate=2)
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(20):
intents.append(
TransmitIntent('addr1',
'msg',
onSuccess=self.successfulIntent,
onError=self.failedIntent))
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=10) >= hs.delay.remaining()
assert timedelta(milliseconds=9) < hs.delay.remaining()
print('Remaining seconds: %s (%s)' %
(hs.delay.remainingSeconds(), type(hs.delay.remainingSeconds())))
sleep(hs.delay.remainingSeconds())
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
assert 20 == len(getattr(self, 'successes', []))
assert 0 == len(getattr(self, 'fails', []))
def testTwoSendsIntentTimeoutIgnored(self):
self.sends = []
hs = HysteresisDelaySender(self.send,
hysteresis_min_period = timedelta(milliseconds=100),
hysteresis_max_period = timedelta(milliseconds=110),
hysteresis_rate = 2)
intent1 = TransmitIntent('addr1', 'msg1')
intent2 = TransmitIntent('addr1', 'msg2', maxPeriod=timedelta(milliseconds=10))
hs.sendWithHysteresis(intent1)
hs.sendWithHysteresis(intent2)
# First was sent immediately, second is delayed
assert 1 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert timedelta(seconds=0) != hs.delay.remaining()
assert timedelta(milliseconds=110) >= hs.delay.remaining()
assert timedelta(milliseconds=95) < hs.delay.remaining()
print('Remaining seconds: %s (%s)'%(hs.delay.remainingSeconds(),
type(hs.delay.remainingSeconds())))
sleep(hs.delay.remainingSeconds())
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
assert intent1 == self.sends[0]
assert intent2 == self.sends[1]
def testTwentySendsSameAddressSameMessageTypeSendAfterDelay(self):
self.sends = []
hs = HysteresisDelaySender(self.send,
hysteresis_min_period = timedelta(milliseconds=2),
hysteresis_max_period = timedelta(milliseconds=20),
hysteresis_rate = 2)
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(20):
intents.append(TransmitIntent('addr1', 'msg'))
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
# The hysteresis delay should be maxed out
t1 = hs.delay.remaining()
assert timedelta(seconds=0) != t1
assert timedelta(milliseconds=20) >= t1
assert timedelta(milliseconds=9) < t1
# Wait the delay period and then check, which should send the
# (latest) queued messages
sleep(hs.delay.remainingSeconds())
hs.checkSends()
# Verify that hysteresis delay is not yet back to zero and
# additional sends are still blocked.
assert not hs.delay.expired() # got refreshed and reduced in checkSends
hs.sendWithHysteresis(intents[0])
assert 2 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
# Verify that the hysteresis delay keeps dropping and
# eventually gets back to zero. After a drop, any pending
# sends that were blocked should be sent.
nsent = 2 # after first wait, checkSends will send the one just queued...
for x in range(100): # don't loop forever
if hs.delay.expired(): break
t2 = hs.delay.remaining()
assert timedelta(seconds=0) != t2
assert t2 < t1
assert nsent == len(getattr(self, 'sends', []))
sleep(hs.delay.remainingSeconds())
t1 = t2
hs.checkSends()
if nsent == 2: nsent = 3
# Now verify hysteresis sender is back to the original state
assert 3 == len(getattr(self, 'sends', []))
hs.sendWithHysteresis(intents[1])
assert 4 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
assert intents[0] == self.sends[2]
assert intents[1] == self.sends[3]
def __init__(self, *args, **kw):
super(ConventioneerAdmin, self).__init__(*args, **kw)
self._cstate = LocalConventionState(
self.myAddress, self.capabilities, self._sCBStats,
getattr(self.transport, 'getConventionAddress', lambda c: None))
self._hysteresisSender = HysteresisDelaySender(self._send_intent)
class ConventioneerAdmin(GlobalNamesAdmin):
"""Extends the AdminCore+GlobalNamesAdmin with ActorSystem Convention
functionality to support multi-host configurations.
"""
def __init__(self, *args, **kw):
super(ConventioneerAdmin, self).__init__(*args, **kw)
self._cstate = LocalConventionState(
self.myAddress, self.capabilities, self._sCBStats,
getattr(self.transport, 'getConventionAddress', lambda c: None))
self._hysteresisSender = HysteresisDelaySender(self._send_intent)
def _updateStatusResponse(self, resp):
self._cstate.updateStatusResponse(resp)
super(ConventioneerAdmin, self)._updateStatusResponse(resp)
def _activate(self):
# Called internally when this ActorSystem has been initialized
# and should be activated for operations.
super(ConventioneerAdmin, self)._activate()
if self.isShuttingDown(): return
self._performIO(self._cstate.setup_convention(True))
def h_ConventionInvite(self, envelope):
if self.isShuttingDown(): return
return self._performIO(
self._cstate.got_convention_invite(envelope.sender))
def h_ConventionRegister(self, envelope):
if self.isShuttingDown(): return
return self._performIO(
self._cstate.got_convention_register(envelope.message))
def h_ConventionDeRegister(self, envelope):
return self._performIO(
self._cstate.got_convention_deregister(envelope.message))
def h_SystemShutdown(self, envelope):
self._performIO(self._cstate.got_system_shutdown())
return super(ConventioneerAdmin, self).h_SystemShutdown(envelope)
return True
def _performIO(self, iolist):
rval = True
for msg in iolist:
if isinstance(msg, HysteresisCancel):
self._hysteresisSender.cancelSends(msg.cancel_addr)
rval = False
elif isinstance(msg, HysteresisSend):
#self._send_intent(msg)
self._hysteresisSender.sendWithHysteresis(msg)
rval = False
elif isinstance(msg, LogAggregator):
if getattr(self, 'asLogger', None):
thesplog('Setting log aggregator of %s to %s',
self.asLogger, msg.aggregatorAddress)
self._send_intent(TransmitIntent(self.asLogger, msg))
elif isinstance(msg, LostRemote):
if hasattr(self.transport, 'lostRemote'):
self.transport.lostRemote(msg.lost_addr)
else:
self._send_intent(msg)
return rval
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
transport_continue = True
try:
while not getattr(self, 'shutdown_completed', False) and \
not isinstance(transport_continue, Thespian__Run_Terminated):
ct = currentTime()
delay = min(
self._cstate.convention_inattention_delay(ct),
ExpirationTimer(None).view(ct)
if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
transport_continue = self.transport.run(
self.handleIncoming, delay.remaining())
# Check Convention status based on the elapsed time
self._performIO(self._cstate.check_convention())
self._hysteresisSender.checkSends()
self._remove_expired_sources()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
# ---- Source Hash Transfers --------------------------------------------------
def h_SourceHashTransferRequest(self, envelope):
sourceHash = envelope.message.sourceHash
src = self._sources.get(sourceHash, None)
if not src or not src.source_valid:
self._send_intent(
TransmitIntent(envelope.sender,
SourceHashTransferReply(sourceHash)))
else:
# Older requests did not have the prefer_original field;
# maintain backward compatibility
orig = getattr(envelope.message, 'prefer_original', False)
self._send_intent(
TransmitIntent(
envelope.sender,
SourceHashTransferReply(
sourceHash,
src.orig_data if orig else src.zipsrc,
src.srcInfo,
original_form=orig)))
return True
def h_SourceHashTransferReply(self, envelope):
sourceHash = envelope.message.sourceHash
if sourceHash not in self._sources:
return True
if envelope.message.isValid():
# nb.. original_form added; use getattr for backward compatibility
if getattr(envelope.message, 'original_form', False):
if self._sourceAuthority:
self._send_intent(
TransmitIntent(
self._sourceAuthority,
ValidateSource(
sourceHash, envelope.message.sourceData,
getattr(envelope.message, 'sourceInfo',
None))))
return True
else:
self._loadValidatedActorSource(
sourceHash,
envelope.message.sourceData,
# sourceInfo added; backward compat.
getattr(envelope.message, 'sourceInfo', None))
return True
self._cancel_pending_actors(self._sources[sourceHash].pending_actors)
del self._sources[sourceHash]
return True
def h_ValidateSource(self, envelope):
rval = True
if not envelope.message.sourceData and \
envelope.sender != self._cstate.conventionLeaderAddr:
# Propagate source unload requests to all convention members
rval = self._performIO(
self._cstate.send_to_all_members(
envelope.message,
# Do not propagate if this is where the
# notification came from; prevents indefinite
# bouncing of this message as long as the
# convention structure is a DAG.
[envelope.sender]))
super(ConventioneerAdmin, self).h_ValidateSource(envelope)
return rval
def _acceptsRemoteLoadedSourcesFrom(self, pendingActorEnvelope):
allowed = self.capabilities.get('AllowRemoteActorSources', 'yes')
return allowed.lower() == 'yes' or \
(allowed == 'LeaderOnly' and
pendingActorEnvelope.sender == self._cstate.conventionLeaderAddr)
# ---- Remote Actor interactions ----------------------------------------------
def _not_compatible(self, createActorEnvelope):
# Called when the current Actor System is not compatible with
# the Actor's actorSystemCapabilityCheck. Forward this
# createActor request to another system that it's compatible
# with.
sourceHash = createActorEnvelope.message.sourceHash
childRequirements = createActorEnvelope.message.targetActorReq
childCName = createActorEnvelope.message.actorClassName
childClass = actualActorClass(
childCName,
partial(loadModuleFromHashSource, sourceHash, self._sources)
if sourceHash else None)
acceptsCaps = lambda caps: checkActorCapabilities(
childClass, caps, childRequirements)
if createActorEnvelope.message.forActor is None:
# Request from external; use sender address
createActorEnvelope.message.forActor = createActorEnvelope.sender
iolist = self._cstate.forward_pending_to_remote_system(
childClass, createActorEnvelope, sourceHash, acceptsCaps)
if iolist:
for each in iolist:
# Expected to be only one; if the transmit fails,
# route it back here so that the next possible
# remote can be tried.
each.addCallback(onFailure=self._pending_send_failed)
each.orig_create_envelope = createActorEnvelope
return self._performIO(iolist)
self._sendPendingActorResponse(
createActorEnvelope,
None,
errorCode=PendingActorResponse.ERROR_No_Compatible_ActorSystem,
errorStr="")
# self._retryPendingChildOperations(childInstance, None)
return True
def _get_missing_source_for_hash(self, sourceHash, createActorEnvelope):
# If this request was forwarded by a remote Admin and the
# sourceHash is not known locally, request it from the sending
# remote Admin
if self._cstate.sentByRemoteAdmin(createActorEnvelope) and \
self._acceptsRemoteLoadedSourcesFrom(createActorEnvelope):
self._sources[sourceHash] = PendingSource(sourceHash, None)
self._sources[sourceHash].pending_actors.append(
createActorEnvelope)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(
createActorEnvelope.sender,
SourceHashTransferRequest(sourceHash,
bool(self._sourceAuthority))))
# sent with hysteresis, so break out to local _run
return False
# No remote Admin to send the source, so fail as normal.
return super(ConventioneerAdmin, self)._get_missing_source_for_hash(
sourceHash, createActorEnvelope)
def _pending_send_failed(self, result, intent):
self._not_compatible(intent.orig_create_envelope)
def h_NotifyOnSystemRegistration(self, envelope):
if envelope.message.enableNotification:
return self._performIO(
self._cstate.add_notification_handler(envelope.sender))
self._cstate.remove_notification_handler(envelope.sender)
return True
def h_PoisonMessage(self, envelope):
self._cstate.remove_notification_handler(envelope.sender)
def _handleChildExited(self, childAddr):
self._cstate.remove_notification_handler(childAddr)
return super(ConventioneerAdmin, self)._handleChildExited(childAddr)
def h_CapabilityUpdate(self, envelope):
msg = envelope.message
updateLocals = self._updSystemCapabilities(msg.capabilityName,
msg.capabilityValue)
rval = True
if not self.isShuttingDown():
rval = self._performIO(
self._cstate.capabilities_have_changed(self.capabilities))
if updateLocals:
self._capUpdateLocalActors()
return rval
class ConventioneerAdmin(GlobalNamesAdmin):
"""Extends the AdminCore+GlobalNamesAdmin with ActorSystem Convention
functionality to support multi-host configurations.
"""
def __init__(self, *args, **kw):
super(ConventioneerAdmin, self).__init__(*args, **kw)
self._cstate = LocalConventionState(
self.myAddress, self.capabilities, self._sCBStats,
getattr(self.transport, 'getConventionAddress', lambda c: None))
self._pendingSources = {
} # key = sourceHash, value is array of PendingActor requests
self._hysteresisSender = HysteresisDelaySender(self._send_intent)
def _updateStatusResponse(self, resp):
self._cstate.updateStatusResponse(resp)
super(ConventioneerAdmin, self)._updateStatusResponse(resp)
def _activate(self):
# Called internally when this ActorSystem has been initialized
# and should be activated for operations.
if self.isShuttingDown(): return
self._performIO(self._cstate.setup_convention(True))
def h_ConventionInvite(self, envelope):
if self.isShuttingDown(): return
#self._performIO(self._cstate.got_convention_invite(envelope.sender))
def h_ConventionRegister(self, envelope):
if self.isShuttingDown(): return
self._performIO(self._cstate.got_convention_register(envelope.message))
def h_ConventionDeRegister(self, envelope):
self._performIO(
self._cstate.got_convention_deregister(envelope.message))
def h_SystemShutdown(self, envelope):
self._performIO(self._cstate.got_system_shutdown())
return super(ConventioneerAdmin, self).h_SystemShutdown(envelope)
def _performIO(self, iolist):
for msg in iolist:
if isinstance(msg, HysteresisCancel):
self._hysteresisSender.cancelSends(msg.cancel_addr)
elif isinstance(msg, HysteresisSend):
#self._send_intent(msg)
self._hysteresisSender.sendWithHysteresis(msg)
elif isinstance(msg, LogAggregator):
if getattr(self, 'asLogger', None):
thesplog('Setting log aggregator of %s to %s',
self.asLogger, msg)
self._send_intent(TransmitIntent(self.asLogger, msg))
elif isinstance(msg, LostRemote):
if hasattr(self.transport, 'lostRemote'):
self.transport.lostRemote(msg.lost_addr)
else:
self._send_intent(msg)
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
try:
while not getattr(self, 'shutdown_completed', False):
delay = min(
self._cstate.convention_inattention_delay(),
ExpiryTime(None) if self._hysteresisSender.delay.expired()
else self._hysteresisSender.delay)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
r = self.transport.run(self.handleIncoming, delay.remaining())
# Check Convention status based on the elapsed time
self._performIO(self._cstate.check_convention())
self._hysteresisSender.checkSends()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
# ---- Source Hash Transfers --------------------------------------------------
def h_SourceHashTransferRequest(self, envelope):
sourceHash = envelope.message.sourceHash
src = self._sources.get(sourceHash, None)
self._send_intent(
TransmitIntent(
envelope.sender,
SourceHashTransferReply(sourceHash, src and src.zipsrc, src
and src.srcInfo)))
return True
def h_SourceHashTransferReply(self, envelope):
sourceHash = envelope.message.sourceHash
pending = self._pendingSources[sourceHash]
del self._pendingSources[sourceHash]
if envelope.message.isValid():
self._sources[sourceHash] = ValidSource(
sourceHash, envelope.message.sourceData,
getattr(envelope.message, 'sourceInfo', None))
for each in pending:
self.h_PendingActor(each)
else:
for each in pending:
self._sendPendingActorResponse(
each,
None,
errorCode=PendingActorResponse.ERROR_Invalid_SourceHash)
return True
def h_ValidateSource(self, envelope):
if not envelope.message.sourceData and \
envelope.sender != self._cstate.conventionLeaderAddr:
# Propagate source unload requests to all convention members
self._performIO(
self._cstate.send_to_all_members(
envelope.message,
# Do not propagate if this is where the
# notification came from; prevents indefinite
# bouncing of this message as long as the
# convention structure is a DAG.
[envelope.sender]))
super(ConventioneerAdmin, self).h_ValidateSource(envelope)
return False # might have sent with hysteresis, so break out to local _run
def _acceptsRemoteLoadedSourcesFrom(self, pendingActorEnvelope):
allowed = self.capabilities.get('AllowRemoteActorSources', 'yes')
return allowed.lower() == 'yes' or \
(allowed == 'LeaderOnly' and
pendingActorEnvelope.sender == self.conventionLeaderAddr)
# ---- Remote Actor interactions ----------------------------------------------
def h_PendingActor(self, envelope):
sourceHash = envelope.message.sourceHash
childRequirements = envelope.message.targetActorReq
thesplog('Pending Actor request received for %s%s reqs %s from %s',
envelope.message.actorClassName,
' (%s)' % sourceHash if sourceHash else '', childRequirements,
envelope.sender)
# If this request was forwarded by a remote Admin and the
# sourceHash is not known locally, request it from the sending
# remote Admin
if sourceHash and \
sourceHash not in self._sources and \
self._cstate.sentByRemoteAdmin(envelope) and \
self._acceptsRemoteLoadedSourcesFrom(envelope):
requestedAlready = self._pendingSources.get(sourceHash, False)
self._pendingSources.setdefault(sourceHash, []).append(envelope)
if not requestedAlready:
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(envelope.sender,
SourceHashTransferRequest(sourceHash)))
return False # sent with hysteresis, so break out to local _run
return True
# If the requested ActorClass is compatible with this
# ActorSystem, attempt to start it, otherwise forward the
# request to any known compatible ActorSystem.
childClass = envelope.message.actorClassName
try:
childClass = actualActorClass(
envelope.message.actorClassName,
partial(loadModuleFromHashSource, sourceHash, self._sources)
if sourceHash else None)
acceptsCaps = lambda caps: checkActorCapabilities(
childClass, caps, childRequirements)
if not acceptsCaps(self.capabilities):
if envelope.message.forActor is None:
# Request from external; use sender address
envelope.message.forActor = envelope.sender
iolist = self._cstate.forward_pending_to_remote_system(
childClass, envelope, sourceHash, acceptsCaps)
for each in iolist:
# Expected to be only one; if the transmit fails,
# route it back here so that the next possible
# remote can be tried.
each.addCallback(onFailure=self._pending_send_failed)
self._performIO(iolist)
return True
except NoCompatibleSystemForActor as ex:
thesplog(str(ex), level=logging.WARNING, primary=True)
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_No_Compatible_ActorSystem)
return True
except InvalidActorSourceHash:
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Invalid_SourceHash)
return True
except InvalidActorSpecification:
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass)
return True
except ImportError as ex:
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Import,
errorStr=str(ex))
return True
except AttributeError as ex:
# Usually when the module has no attribute FooActor
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass,
errorStr=str(ex))
return True
except Exception as ex:
import traceback
thesplog('Exception "%s" handling PendingActor: %s',
ex,
traceback.format_exc(),
level=logging.ERROR)
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass,
errorStr=str(ex))
return True
return super(ConventioneerAdmin, self).h_PendingActor(envelope)
def _pending_send_failed(self, result, intent):
self.h_PendingActor(
ReceiveEnvelope(msg=intent.message, sender=self.myAddress))
def h_NotifyOnSystemRegistration(self, envelope):
if envelope.message.enableNotification:
self._performIO(
self._cstate.add_notification_handler(envelope.sender))
else:
self._cstate.remove_notification_handler(envelope.sender)
return True
def h_PoisonMessage(self, envelope):
self._cstate.remove_notification_handler(envelope.sender)
def _handleChildExited(self, childAddr):
self._cstate.remove_notification_handler(childAddr)
return super(ConventioneerAdmin, self)._handleChildExited(childAddr)
def h_CapabilityUpdate(self, envelope):
msg = envelope.message
updateLocals = self._updSystemCapabilities(msg.capabilityName,
msg.capabilityValue)
if not self.isShuttingDown():
self._performIO(
self._cstate.capabilities_have_changed(self.capabilities))
if updateLocals:
self._capUpdateLocalActors()
return False # might have sent with Hysteresis, so return to _run loop here
class ConventioneerAdmin(GlobalNamesAdmin):
"""Extends the AdminCore+GlobalNamesAdmin with ActorSystem Convention
functionality to support multi-host configurations.
"""
def __init__(self, *args, **kw):
super(ConventioneerAdmin, self).__init__(*args, **kw)
self._cstate = LocalConventionState(
self.myAddress,
self.capabilities,
self._sCBStats,
getattr(self.transport, 'getConventionAddress', lambda c: None))
self._hysteresisSender = HysteresisDelaySender(self._send_intent)
def _updateStatusResponse(self, resp):
self._cstate.updateStatusResponse(resp)
super(ConventioneerAdmin, self)._updateStatusResponse(resp)
def _activate(self):
# Called internally when this ActorSystem has been initialized
# and should be activated for operations.
super(ConventioneerAdmin, self)._activate()
if self.isShuttingDown(): return
self._performIO(self._cstate.setup_convention(True))
def h_ConventionInvite(self, envelope):
if self.isShuttingDown(): return
self._performIO(self._cstate.got_convention_invite(envelope.sender))
return True
def h_ConventionRegister(self, envelope):
if self.isShuttingDown(): return
self._performIO(self._cstate.got_convention_register(envelope.message))
return True
def h_ConventionDeRegister(self, envelope):
self._performIO(self._cstate.got_convention_deregister(envelope.message))
return True
def h_SystemShutdown(self, envelope):
self._performIO(self._cstate.got_system_shutdown())
return super(ConventioneerAdmin, self).h_SystemShutdown(envelope)
return True
def _performIO(self, iolist):
for msg in iolist:
if isinstance(msg, HysteresisCancel):
self._hysteresisSender.cancelSends(msg.cancel_addr)
elif isinstance(msg, HysteresisSend):
#self._send_intent(msg)
self._hysteresisSender.sendWithHysteresis(msg)
elif isinstance(msg, LogAggregator):
if getattr(self, 'asLogger', None):
thesplog('Setting log aggregator of %s to %s', self.asLogger, msg.aggregatorAddress)
self._send_intent(TransmitIntent(self.asLogger, msg))
elif isinstance(msg, LostRemote):
if hasattr(self.transport, 'lostRemote'):
self.transport.lostRemote(msg.lost_addr)
else:
self._send_intent(msg)
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
transport_continue = True
try:
while not getattr(self, 'shutdown_completed', False) and \
not isinstance(transport_continue, Thespian__Run_Terminated):
delay = min(self._cstate.convention_inattention_delay(),
ExpirationTimer(None) if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay
)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
transport_continue = self.transport.run(self.handleIncoming,
delay.remaining())
# Check Convention status based on the elapsed time
self._performIO(self._cstate.check_convention())
self._hysteresisSender.checkSends()
self._remove_expired_sources()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(),
level=logging.ERROR, exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
# ---- Source Hash Transfers --------------------------------------------------
def h_SourceHashTransferRequest(self, envelope):
sourceHash = envelope.message.sourceHash
src = self._sources.get(sourceHash, None)
if not src or not src.source_valid:
self._send_intent(
TransmitIntent(envelope.sender,
SourceHashTransferReply(sourceHash)))
else:
# Older requests did not have the prefer_original field;
# maintain backward compatibility
orig = getattr(envelope.message, 'prefer_original', False)
self._send_intent(
TransmitIntent(
envelope.sender,
SourceHashTransferReply(
sourceHash,
src.orig_data if orig else src.zipsrc,
src.srcInfo,
original_form = orig)))
return True
def h_SourceHashTransferReply(self, envelope):
sourceHash = envelope.message.sourceHash
if sourceHash not in self._sources:
return True
if envelope.message.isValid():
# nb.. original_form added; use getattr for backward compatibility
if getattr(envelope.message, 'original_form', False):
if self._sourceAuthority:
self._send_intent(
TransmitIntent(
self._sourceAuthority,
ValidateSource(sourceHash,
envelope.message.sourceData,
getattr(envelope.message,
'sourceInfo', None))))
return True
else:
self._loadValidatedActorSource(sourceHash,
envelope.message.sourceData,
# sourceInfo added; backward compat.
getattr(envelope.message,
'sourceInfo', None))
return True
self._cancel_pending_actors(self._sources[sourceHash].pending_actors)
del self._sources[sourceHash]
return True
def h_ValidateSource(self, envelope):
if not envelope.message.sourceData and \
envelope.sender != self._cstate.conventionLeaderAddr:
# Propagate source unload requests to all convention members
self._performIO(
self._cstate.send_to_all_members(
envelope.message,
# Do not propagate if this is where the
# notification came from; prevents indefinite
# bouncing of this message as long as the
# convention structure is a DAG.
[envelope.sender]))
super(ConventioneerAdmin, self).h_ValidateSource(envelope)
return False # might have sent with hysteresis, so break out to local _run
def _acceptsRemoteLoadedSourcesFrom(self, pendingActorEnvelope):
allowed = self.capabilities.get('AllowRemoteActorSources', 'yes')
return allowed.lower() == 'yes' or \
(allowed == 'LeaderOnly' and
pendingActorEnvelope.sender == self._cstate.conventionLeaderAddr)
# ---- Remote Actor interactions ----------------------------------------------
def h_PendingActor(self, envelope):
sourceHash = envelope.message.sourceHash
childRequirements = envelope.message.targetActorReq
thesplog('Pending Actor request received for %s%s reqs %s from %s',
envelope.message.actorClassName,
' (%s)'%sourceHash if sourceHash else '',
childRequirements, envelope.sender)
if sourceHash:
if sourceHash not in self._sources:
# If this request was forwarded by a remote Admin and the
# sourceHash is not known locally, request it from the sending
# remote Admin
if self._cstate.sentByRemoteAdmin(envelope) and \
self._acceptsRemoteLoadedSourcesFrom(envelope):
self._sources[sourceHash] = PendingSource(sourceHash, None)
self._sources[sourceHash].pending_actors.append(envelope)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(
envelope.sender,
SourceHashTransferRequest(sourceHash,
bool(self._sourceAuthority))))
# sent with hysteresis, so break out to local _run
return False
if sourceHash in self._sources and \
not self._sources[sourceHash].source_valid:
# Still pending, add this create request to the waiting list
self._sources[sourceHash].pending_actors.append(envelope)
return True
# If the requested ActorClass is compatible with this
# ActorSystem, attempt to start it, otherwise forward the
# request to any known compatible ActorSystem.
childClass = envelope.message.actorClassName
try:
childClass = actualActorClass(envelope.message.actorClassName,
partial(loadModuleFromHashSource,
sourceHash,
self._sources)
if sourceHash else None)
acceptsCaps = lambda caps: checkActorCapabilities(childClass, caps,
childRequirements)
if not acceptsCaps(self.capabilities):
if envelope.message.forActor is None:
# Request from external; use sender address
envelope.message.forActor = envelope.sender
iolist = self._cstate.forward_pending_to_remote_system(
childClass, envelope, sourceHash, acceptsCaps)
for each in iolist:
# Expected to be only one; if the transmit fails,
# route it back here so that the next possible
# remote can be tried.
each.addCallback(onFailure=self._pending_send_failed)
self._performIO(iolist)
return True
except NoCompatibleSystemForActor as ex:
thesplog(str(ex), level=logging.WARNING, primary=True)
self._sendPendingActorResponse(
envelope, None,
errorCode=PendingActorResponse.ERROR_No_Compatible_ActorSystem)
return True
except InvalidActorSourceHash:
self._sendPendingActorResponse(
envelope, None,
errorCode=PendingActorResponse.ERROR_Invalid_SourceHash)
return True
except InvalidActorSpecification as ex:
thesplog('Error: InvalidActorSpecification: %s', str(ex), exc_info=True)
self._sendPendingActorResponse(
envelope, None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass,
errorStr=str(ex))
return True
except ImportError as ex:
self._sendPendingActorResponse(
envelope, None,
errorCode=PendingActorResponse.ERROR_Import,
errorStr=str(ex))
return True
except AttributeError as ex:
# Usually when the module has no attribute FooActor
thesplog('Error: AttributeError: %s', str(ex), exc_info=True)
self._sendPendingActorResponse(
envelope, None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass,
errorStr=str(ex))
return True
except Exception as ex:
import traceback
thesplog('Exception "%s" handling PendingActor: %s', ex, traceback.format_exc(), level=logging.ERROR)
self._sendPendingActorResponse(
envelope, None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass,
errorStr=str(ex))
return True
return super(ConventioneerAdmin, self).h_PendingActor(envelope)
def _pending_send_failed(self, result, intent):
self.h_PendingActor(ReceiveEnvelope(msg=intent.message, sender=self.myAddress))
def h_NotifyOnSystemRegistration(self, envelope):
if envelope.message.enableNotification:
self._performIO(
self._cstate.add_notification_handler(envelope.sender))
else:
self._cstate.remove_notification_handler(envelope.sender)
return True
def h_PoisonMessage(self, envelope):
self._cstate.remove_notification_handler(envelope.sender)
def _handleChildExited(self, childAddr):
self._cstate.remove_notification_handler(childAddr)
return super(ConventioneerAdmin, self)._handleChildExited(childAddr)
def h_CapabilityUpdate(self, envelope):
msg = envelope.message
updateLocals = self._updSystemCapabilities(msg.capabilityName,
msg.capabilityValue)
if not self.isShuttingDown():
self._performIO(
self._cstate.capabilities_have_changed(self.capabilities))
if updateLocals:
self._capUpdateLocalActors()
return False # might have sent with Hysteresis, so return to _run loop here
def testTwentySendsSameAddressSameMessageTypeSendAfterDelay(self):
self.sends = []
min_h_period = timedelta(milliseconds=2)
max_h_period = timedelta(milliseconds=20)
h_rate = 2
hs = HysteresisDelaySender(self.send,
hysteresis_min_period=min_h_period,
hysteresis_max_period=max_h_period,
hysteresis_rate=h_rate)
# Create the messages to send
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(20):
intents.append(TransmitIntent('addr1', 'msg'))
# Send all intents in rapid succession
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
# The hysteresis delay should be maxed out
t1 = hs.delay.remaining()
assert timedelta(seconds=0) != t1
assert max_h_period >= t1
# Wait the delay period and then check, which should send the
# (latest) queued messages. Because all the messages and
# target addresses are identical, this will actually only send
# a single message.
sleep(hs.delay.remainingSeconds())
hs.checkSends()
assert 2 == len(getattr(self, 'sends', []))
# Verify that although the queued messages were sent, the
# hysteresis delay is not yet back to zero and additional
# sends are still blocked.
assert not hs.delay.expired() # refreshed and reduced in checkSends
hs.sendWithHysteresis(intents[0])
t1 = hs.delay.remaining() # send attempt probably bumped this up again
assert 2 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
# Verify that the hysteresis delay keeps dropping and
# eventually gets back to zero. After a drop, any pending
# sends that were blocked should be sent.
nsent = 2 # after first wait, checkSends will send the one just queued...
for x in range(100): # don't loop forever
if hs.delay.expired(): break
t2 = hs.delay.remaining()
assert timedelta(seconds=0) != t2
assert (x, t2) < (x, t1)
assert nsent == len(getattr(self, 'sends', []))
sleep(hs.delay.remainingSeconds())
t1 = t2
hs.checkSends()
if nsent == 2:
# All queued sends should now have been sent, but
# since they are all the same, there was only one more
# actual send.
nsent = 3
# Now verify hysteresis sender is back to the original state
assert 3 == len(getattr(self, 'sends', []))
hs.sendWithHysteresis(intents[1])
assert 4 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
assert intents[0] == self.sends[2]
assert intents[1] == self.sends[3]
# Verify that all intents got completed even though some were
# duplicates and not actually sent.
for each in intents:
assert each.result == SendStatus.Sent
def testTwentySendsSameAddressSameMessageTypeSendAfterDelay(self):
self.sends = []
hs = HysteresisDelaySender(
self.send,
hysteresis_min_period=timedelta(milliseconds=2),
hysteresis_max_period=timedelta(milliseconds=20),
hysteresis_rate=2)
intents = [TransmitIntent('addr1', 'msg1')]
for num in range(20):
intents.append(TransmitIntent('addr1', 'msg'))
for each in intents:
hs.sendWithHysteresis(each)
# First was sent immediately, all others are delayed
assert 1 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
# The hysteresis delay should be maxed out
t1 = hs.delay.remaining()
assert timedelta(seconds=0) != t1
assert timedelta(milliseconds=20) >= t1
assert timedelta(milliseconds=9) < t1
# Wait the delay period and then check, which should send the
# (latest) queued messages
sleep(hs.delay.remainingSeconds())
hs.checkSends()
# Verify that hysteresis delay is not yet back to zero and
# additional sends are still blocked.
assert not hs.delay.expired(
) # got refreshed and reduced in checkSends
hs.sendWithHysteresis(intents[0])
assert 2 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
# Verify that the hysteresis delay keeps dropping and
# eventually gets back to zero. After a drop, any pending
# sends that were blocked should be sent.
nsent = 2 # after first wait, checkSends will send the one just queued...
for x in range(100): # don't loop forever
if hs.delay.expired(): break
t2 = hs.delay.remaining()
assert timedelta(seconds=0) != t2
assert t2 < t1
assert nsent == len(getattr(self, 'sends', []))
sleep(hs.delay.remainingSeconds())
t1 = t2
hs.checkSends()
if nsent == 2: nsent = 3
# Now verify hysteresis sender is back to the original state
assert 3 == len(getattr(self, 'sends', []))
hs.sendWithHysteresis(intents[1])
assert 4 == len(getattr(self, 'sends', []))
assert intents[0] == self.sends[0]
assert intents[-1] == self.sends[1]
assert intents[0] == self.sends[2]
assert intents[1] == self.sends[3]