class LocalConventionState(object):
# The general process of leader management in an HA configuration
# where there are multiple potential leaders is currently:
#
# 1. The self._conventionAddress is a list of potential leader
# addresses. This is set from the capabilities (with an assist
# from the transport to map the addresses to a
# transport-specific address). At present, it is assumed that
# all convention members are initialized with the same list,
# and in the same order (excluding invite-only members).
#
# 2. The current active leader is the "highest" leader: the one
# with the lowest index in the list (appears first) that is
# also currently active (as provided by the
# self._conventionMembers list that is already updated by
# active registrations and either active deregistrations or
# timeouts. The self._conventionLeaderIdx is a helper to
# indicate the current active leader without searching the
# self._conventionMembers array.
#
# 3. The standard operational mode of a convention is that the
# leader is largely passive in terms of membership: members
# initially join and subsequently periodically check-in by
# sending a registration request (eliciting a corresponding
# response from the leader). The leader removes them from the
# convention if they don't check-in within a specified period
# of time, but does not actively probe the member. The
# principle behind this is that traffic should only be
# generated for active members and not inactive members.
#
# 4. With the addition of HA support, the member registration and
# check-in is always sent to *all* potential leaders,
# regardless of which is thought to be the current active
# leader by that member. This includes all potential leaders,
# which send a registration to potential leaders higher than
# themselves.
#
# 5. When a potential leader receives a check-in registration, it
# will check to see if it believes itself to be the
# highest-priority active leader. If so, it will respond and
# the remote will see that it is the current leader (including
# any other potential leaders, active or not). Potential
# leaders that see a higher-priority leader will not respond to
# a check-in request, but will have updated their internal
# member information list.
#
# Based on the above, a leadership transition occurs naturally
# (albeit slowly) through the passive combination of #2, #4, and
# #5. At present, there is no exchange of state information
# between leaders, so any context maintained by one leader will be
# lost in moving to a new leader [this is an area that should be
# improved in future work]
def __init__(self, myAddress, capabilities, sCBStats,
getConventionAddressFunc):
self._myAddress = myAddress
self._capabilities = capabilities
self._sCBStats = sCBStats
self._conventionMembers = AssocList(
) # key=Remote Admin Addr, value=ConventionMemberData
self._conventionNotificationHandlers = []
self._getConventionAddr = getConventionAddressFunc
self._conventionLeaderIdx = 0
self._conventionAddress = getConventionAddressFunc(capabilities)
if not isinstance(self._conventionAddress, list):
self._conventionAddress = [self._conventionAddress]
self._conventionRegistration = ExpirationTimer(
CONVENTION_REREGISTRATION_PERIOD)
self._has_been_activated = False
self._invited = False # entered convention as a result of an explicit invite
@property
def myAddress(self):
return self._myAddress
@property
def capabilities(self):
return self._capabilities
def updateStatusResponse(self, resp):
resp.setConventionLeaderAddress(self.conventionLeaderAddr)
resp.setConventionRegisterTime(self._conventionRegistration)
for each in self._conventionMembers.values():
resp.addConventioneer(each.remoteAddress, each.registryValid)
resp.setNotifyHandlers(self._conventionNotificationHandlers)
def active_in_convention(self):
# If this is the convention leader, it is automatically
# active, otherwise this convention member should have a
# convention leader and that leader should have an active
# entry in the _conventionMembers table (indicating it has
# updated this system with its information)
return bool(
self.conventionLeaderAddr
and self._conventionMembers.find(self.conventionLeaderAddr))
@property
def conventionLeaderAddr(self):
return self._conventionAddress[self._conventionLeaderIdx]
def isConventionLeader(self):
"Return true if this is the current leader of this convention"
# This checks to see if the current system is the convention
# leader. This check is dynamic and may have the effect of
# changing the determination of which is the actual convention
# leader.
if self._conventionAddress == [None]:
return True
for (idx, myLeader) in enumerate(self._conventionAddress):
if myLeader == self.myAddress:
# I am the highest active leader, therefore I am the
# current actual leader.
self._conventionLeaderIdx = idx
return True
if self._conventionMembers.find(myLeader):
# A leader higher priority than myself exists (this is
# actually the highest active due to the processing
# order), so it's the leader, not me.
self._conventionLeaderIdx = idx
return False
return False
def capabilities_have_changed(self, new_capabilities):
self._capabilities = new_capabilities
return self.setup_convention()
def setup_convention(self, activation=False):
"""Called to perform the initial registration with the convention
leader (unless this *is* the leader) and also whenever
connectivity to the convention leader is restored.
Performs some administration and then attempts to register
with the convention leader.
"""
self._has_been_activated |= activation
rmsgs = []
# If not specified in capabilities, don't override any invites
# that may have been received.
self._conventionAddress = self._getConventionAddr(self.capabilities) or \
self._conventionAddress
if not isinstance(self._conventionAddress, list):
self._conventionAddress = [self._conventionAddress]
if self._conventionLeaderIdx >= len(self._conventionAddress):
self._conventionLeaderIdx = 0
leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \
if self.conventionLeaderAddr else True
# Register with all other leaders to notify them that this potential leader is online
if self._conventionAddress and \
self._conventionAddress[0] != None:
for possibleLeader in self._conventionAddress:
if possibleLeader == self.myAddress:
# Don't register with myself
continue
re_registering = not leader_is_gone and \
(possibleLeader == self.conventionLeaderAddr)
thesplog(
'Admin registering with Convention @ %s (%s)',
possibleLeader,
'first time' if not re_registering else 're-registering',
level=logging.INFO,
primary=True)
rmsgs.append(
HysteresisSend(possibleLeader,
ConventionRegister(self.myAddress,
self.capabilities,
not re_registering),
onSuccess=self._setupConventionCBGood,
onError=self._setupConventionCBError))
self._conventionRegistration = ExpirationTimer(
CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def _setupConventionCBGood(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registered')
if hasattr(self, '_conventionLeaderMissCount'):
delattr(self, '_conventionLeaderMissCount')
def _setupConventionCBError(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registration Failed')
if hasattr(self, '_conventionLeaderMissCount'):
self._conventionLeaderMissCount += 1
else:
self._conventionLeaderMissCount = 1
thesplog('Admin cannot register with convention @ %s (miss %d): %s',
finishedIntent.targetAddr,
self._conventionLeaderMissCount,
result,
level=logging.WARNING,
primary=True)
def got_convention_invite(self, sender):
self._conventionAddress = [sender]
self._conventionLeaderIdx = 0
self._invited = True
return self.setup_convention()
def got_convention_register(self, regmsg):
# Called when remote convention member has sent a
# ConventionRegister message. This is first called the leader
# when the member registers with the leader, and then on the
# member when the leader responds with same. Thus the current
# node could be a member, a potential leader, the current
# leader, or a potential leader with higher potential than the
# current leader and which should become the new leader.
self._sCBStats.inc('Admin Handle Convention Registration')
if self._invited and not self.conventionLeaderAddr:
# Lost connection to an invitation-only convention.
# Cannot join again until another invitation is received.
return []
# Remote member may re-register if changing capabilities
rmsgs = []
registrant = regmsg.adminAddress
prereg = getattr(regmsg, 'preRegister',
False) # getattr used; see definition
existing = self._conventionMembers.find(registrant)
thesplog('Got Convention %sregistration from %s (%s) (new? %s)',
'pre-' if prereg else '',
registrant,
'first time' if regmsg.firstTime else 're-registering',
not existing,
level=logging.DEBUG)
if registrant == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog(
'Convention registration from %s is an invalid address; ignoring.',
registrant,
level=logging.WARNING)
return rmsgs
existingPreReg = (
# existing.preRegOnly
# or existing.preRegistered
existing.permanentEntry) if existing else False
notify = (not existing or existing.preRegOnly) and not prereg
if regmsg.firstTime or not existing:
if existing:
existing = None
notify = not prereg
rmsgs.extend(self._remote_system_cleanup(registrant))
newmember = ConventionMemberData(registrant, regmsg.capabilities,
prereg)
if prereg or existingPreReg:
newmember.preRegistered = PreRegistration()
self._conventionMembers.add(registrant, newmember)
else:
existing.refresh(regmsg.capabilities, prereg or existingPreReg)
if not prereg:
existing.preRegOnly = False
if not self.isConventionLeader():
self._conventionRegistration = ExpirationTimer(
CONVENTION_REREGISTRATION_PERIOD)
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
# Convention Members normally periodically initiate a
# membership message, to which the leader confirms by
# responding.
#if self.isConventionLeader() or prereg or regmsg.firstTime:
if prereg:
# If this was a pre-registration, that identifies this
# system as the "leader" for that remote. Also, if the
# remote sent this because it was a pre-registration
# leader, it doesn't yet have all the member information
# so the member should respond.
rmsgs.append(HysteresisCancel(registrant))
rmsgs.append(TransmitIntent(registrant, ConventionInvite()))
elif (self.isConventionLeader() or prereg or regmsg.firstTime or \
(existing and existing.permanentEntry)):
# If we are the Convention Leader, this would be the point to
# inform all other registrants of the new registrant. At
# present, there is no reciprocity here, so just update the
# new registrant with the leader's info.
rmsgs.append(
TransmitIntent(
registrant,
ConventionRegister(self.myAddress, self.capabilities)))
if notify:
rmsgs.extend(
self._notifications_of(
ActorSystemConventionUpdate(registrant,
regmsg.capabilities, True)))
return rmsgs
def _notifications_of(self, msg):
return [
TransmitIntent(H, msg)
for H in self._conventionNotificationHandlers
]
def add_notification_handler(self, addr):
if addr not in self._conventionNotificationHandlers:
self._conventionNotificationHandlers.append(addr)
# Now update the registrant on the current state of all convention members
return [
TransmitIntent(
addr,
ActorSystemConventionUpdate(M.remoteAddress,
M.remoteCapabilities, True))
for M in self._conventionMembers.values() if not M.preRegOnly
]
return []
def remove_notification_handler(self, addr):
self._conventionNotificationHandlers = [
H for H in self._conventionNotificationHandlers if H != addr
]
def got_convention_deregister(self, deregmsg):
self._sCBStats.inc('Admin Handle Convention De-registration')
remoteAdmin = deregmsg.adminAddress
if remoteAdmin == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog(
'Convention deregistration from %s is an invalid address; ignoring.',
remoteAdmin,
level=logging.WARNING)
rmsgs = []
if getattr(deregmsg, 'preRegistered',
False): # see definition for getattr use
existing = self._conventionMembers.find(remoteAdmin)
if existing:
existing.preRegistered = None
rmsgs.append(
TransmitIntent(remoteAdmin,
ConventionDeRegister(self.myAddress)))
return rmsgs + self._remote_system_cleanup(remoteAdmin)
def got_system_shutdown(self):
return self.exit_convention()
def exit_convention(self):
self.invited = False
gen_ops = lambda addr: [
HysteresisCancel(addr),
TransmitIntent(addr, ConventionDeRegister(self.myAddress)),
]
terminate = lambda a: [self._remote_system_cleanup(a), gen_ops(a)][-1]
if self.conventionLeaderAddr and \
self.conventionLeaderAddr != self.myAddress:
thesplog('Admin de-registering with Convention @ %s',
str(self.conventionLeaderAddr),
level=logging.INFO,
primary=True)
# Cache convention leader address because it might get reset by terminate()
claddr = self.conventionLeaderAddr
terminate(self.conventionLeaderAddr)
return gen_ops(claddr)
return join(
fmap(terminate, [
M.remoteAddress for M in self._conventionMembers.values()
if M.remoteAddress != self.myAddress
]))
def check_convention(self):
ct = currentTime()
rmsgs = []
if self._has_been_activated:
rmsgs = foldl(
lambda x, y: x + y, [
self._check_preregistered_ping(ct, member)
for member in self._conventionMembers.values()
],
self._convention_leader_checks(ct)
if self.isConventionLeader() or not self.conventionLeaderAddr
else self._convention_member_checks(ct))
if self._conventionRegistration.view(ct).expired():
self._conventionRegistration = ExpirationTimer(
CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def _convention_leader_checks(self, ct):
return foldl(lambda x, y: x + y, [
self._missed_checkin_remote_cleanup(R) for R in [
member for member in self._conventionMembers.values()
if member.registryValid.view(ct).expired()
]
], [])
def _missed_checkin_remote_cleanup(self, remote_member):
thesplog('%s missed %d checkins (%s); assuming it has died',
str(remote_member),
CONVENTION_REGISTRATION_MISS_MAX,
str(remote_member.registryValid),
level=logging.WARNING,
primary=True)
return self._remote_system_cleanup(remote_member.remoteAddress)
def _convention_member_checks(self, ct):
rmsgs = []
# Re-register with the Convention if it's time
if self.conventionLeaderAddr and \
self._conventionRegistration.view(ct).expired():
if getattr(self, '_conventionLeaderMissCount', 0) >= \
CONVENTION_REGISTRATION_MISS_MAX:
thesplog('Admin convention registration lost @ %s (miss %d)',
self.conventionLeaderAddr,
self._conventionLeaderMissCount,
level=logging.WARNING,
primary=True)
rmsgs.extend(
self._remote_system_cleanup(self.conventionLeaderAddr))
self._conventionLeaderMissCount = 0
else:
rmsgs.extend(self.setup_convention())
return rmsgs
def _check_preregistered_ping(self, ct, member):
if member.preRegistered and \
member.preRegistered.pingValid.view(ct).expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
# If remote misses a checkin, re-extend the
# invitation. This also helps re-initiate a socket
# connection if a TxOnly socket has been lost.
member.preRegistered.pingValid = ExpirationTimer(
convention_reinvite_adjustment(
CONVENTION_RESTART_PERIOD if member.registryValid.view(ct).
expired() else CONVENTION_REREGISTRATION_PERIOD))
return [
HysteresisSend(member.remoteAddress,
ConventionInvite(),
onSuccess=self._preRegQueryNotPending,
onError=self._preRegQueryNotPending)
]
return []
def _preRegQueryNotPending(self, result, finishedIntent):
remoteAddr = finishedIntent.targetAddr
member = self._conventionMembers.find(remoteAddr)
if member and member.preRegistered:
member.preRegistered.pingPending = False
def _remote_system_cleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership. This is also called on
a First Time connection from the remote to discard any
previous connection information.
"""
thesplog('Convention cleanup or deregistration for %s (known? %s)',
registrant,
bool(self._conventionMembers.find(registrant)),
level=logging.INFO)
rmsgs = [LostRemote(registrant)]
cmr = self._conventionMembers.find(registrant)
if not cmr or cmr.preRegOnly:
return []
# Send exited notification to conventionNotificationHandler (if any)
for each in self._conventionNotificationHandlers:
rmsgs.append(
TransmitIntent(
each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
cla = self.conventionLeaderAddr
self._conventionMembers.rmv(registrant)
if registrant == cla:
if self._invited:
# Don't clear invited: once invited, that
# perpetually indicates this should be only a
# member and never a leader.
self._conventionAddress = [None]
else:
rmsgs.extend(self.setup_convention())
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpirationTimer(None)
cmr.preRegOnly = True
return rmsgs + [HysteresisCancel(registrant)]
def sentByRemoteAdmin(self, envelope):
for each in self._conventionMembers.values():
if envelope.sender == each.remoteAddress:
return True
return False
def convention_inattention_delay(self, current_time):
return (self._conventionRegistration or ExpirationTimer(
CONVENTION_REREGISTRATION_PERIOD if self.active_in_convention()
or self.isConventionLeader() else CONVENTION_RESTART_PERIOD)
).view(current_time)
def forward_pending_to_remote_system(self, childClass, envelope,
sourceHash, acceptsCaps):
alreadyTried = getattr(envelope.message, 'alreadyTried', [])
ct = currentTime()
if self.myAddress not in alreadyTried:
# Don't send request back to this actor system: it cannot
# handle it
alreadyTried.append(self.myAddress)
remoteCandidates = [
K for K in self._conventionMembers.values()
if not K.registryValid.view(ct).expired()
and K.remoteAddress != envelope.sender # source Admin
and K.remoteAddress not in alreadyTried
and acceptsCaps(K.remoteCapabilities)
]
if not remoteCandidates:
if self.isConventionLeader() or not self.conventionLeaderAddr:
raise NoCompatibleSystemForActor(
childClass, 'No known ActorSystems can handle a %s for %s',
childClass, envelope.message.forActor)
# Let the Convention Leader try to find an appropriate ActorSystem
bestC = self.conventionLeaderAddr
else:
# distribute equally amongst candidates
C = [(K.remoteAddress, len(K.hasRemoteActors))
for K in remoteCandidates]
bestC = foldl(
lambda best, possible: best
if best[1] <= possible[1] else possible, C)[0]
thesplog('Requesting creation of %s%s on remote admin %s',
envelope.message.actorClassName,
' (%s)' % sourceHash if sourceHash else '', bestC)
if bestC in alreadyTried:
return [] # Have to give up, no-one can handle this
# Don't send request to this remote again, it has already
# been tried. This would also be indicated by that system
# performing the add of self.myAddress as below, but if
# there is disagreement between the local and remote
# addresses, this addition will prevent continual
# bounceback.
alreadyTried.append(bestC)
envelope.message.alreadyTried = alreadyTried
return [TransmitIntent(bestC, envelope.message)]
def send_to_all_members(self, message, exception_list=None):
return [
HysteresisSend(M.remoteAddress, message)
for M in self._conventionMembers.values()
if M.remoteAddress not in (exception_list or [])
]
class MultiprocessQueueTCore_Common(object):
def __init__(self, myQueue, parentQ, adminQ, adminAddr):
self._myInputQ = myQueue
self._parentQ = parentQ
self._adminQ = adminQ
self._adminAddr = adminAddr
# _queues is a map of direct child ActorAddresses to Queue instance. Note
# that there will be multiple keys mapping to the same Queue
# instance because routing is only either to the Parent or to
# an immediate Child.
self._queues = AssocList() # addr -> queue
# _fwdvia represents routing for other than immediate parent
# or child (there may be multiple target addresses mapping to
# the same forward address.
self._fwdvia = AssocList() # targetAddress -> fwdViaAddress
self._deadaddrs = []
# Signals can set these to true; they should be checked and
# reset by the main processing loop. There is a small window
# where they could be missed because signals are not queued,
# but this should handle the majority of situations. Note
# that the Queue object is NOT signal-safe, so don't try to
# queue signals that way.
self._checkChildren = False
self._shutdownSignalled = False
def mainLocalInputQueueEndpoint(self): return self._myInputQ
def adminQueueEndpoint(self): return self._adminQ
@property
def adminAddr(self): return self._adminAddr
def protectedFileNumList(self):
return foldl(lambda a, b: a+[b._reader.fileno(), b._writer.fileno()],
[self._myInputQ, self._parentQ, self._adminQ] +
list(self._queues.values()), [])
def childResetFileNumList(self):
return foldl(lambda a, b: a+[b._reader.fileno(), b._writer.fileno()],
[self._parentQ] +
list(self._queues.values()), [])
def add_endpoint(self, child_addr, child_queue):
self._queues.add(child_addr, child_queue)
def set_address_to_dead(self, child_addr):
self._queues.rmv(child_addr)
self._fwdvia.rmv(child_addr)
self._fwdvia.rmv_value(child_addr)
self._deadaddrs.append(child_addr)
def abort_core_run(self):
self._aborting_run = True
def core_common_transmit(self, transmit_intent, from_addr):
try:
if self.isMyAddress(transmit_intent.targetAddr):
if transmit_intent.message:
self._myInputQ.put( (from_addr, transmit_intent.serMsg),
True,
timePeriodSeconds(transmit_intent.delay()))
else:
tgtQ = self._queues.find(transmit_intent.targetAddr)
if tgtQ:
tgtQ.put((from_addr, transmit_intent.serMsg), True,
timePeriodSeconds(transmit_intent.delay()))
else:
# None means sent by parent, so don't send BACK to parent if unknown
topOrFromBelow = from_addr if self._parentQ else None
(self._parentQ or self._adminQ).put(
(topOrFromBelow, transmit_intent.serMsg),
True,
timePeriodSeconds(transmit_intent.delay()))
transmit_intent.tx_done(SendStatus.Sent)
return
except Q.Full:
pass
transmit_intent.tx_done(SendStatus.DeadTarget if not isinstance(
transmit_intent._message,
(ChildActorExited, ActorExitRequest)) else SendStatus.Failed)
def core_common_receive(self, incoming_handler, for_local_addr, run_time_f):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
if incoming_handler == TransmitOnly or \
isinstance(incoming_handler, TransmitOnly):
# transmits are not queued/multistage in this transport, no waiting
return 0
self._aborting_run = False
while not run_time_f().expired() and not self._aborting_run:
try:
# Unfortunately, the Queue object is not signal-safe,
# so a frequent wakeup is needed to check
# _checkChildren and _shutdownSignalled.
rcvd = self._myInputQ.get(True,
min(run_time_f().remainingSeconds() or
QUEUE_CHECK_PERIOD,
QUEUE_CHECK_PERIOD))
except Q.Empty:
if not self._checkChildren and not self._shutdownSignalled:
# Probably a timeout, but let the while loop decide for sure
continue
rcvd = 'BuMP'
if rcvd == 'BuMP':
relayAddr = sendAddr = destAddr = for_local_addr
if self._checkChildren:
self._checkChildren = False
msg = ChildMayHaveDied()
elif self._shutdownSignalled:
self._shutdownSignalled = False
msg = ActorExitRequest()
else:
return Thespian__UpdateWork()
else:
relayAddr, (sendAddr, destAddr, msg) = rcvd
if not self._queues.find(sendAddr):
# We don't directly know about this sender, so
# remember what path this arrived on to know where to
# direct future messages for this sender.
if relayAddr and self._queues.find(relayAddr) and \
not self._fwdvia.find(sendAddr):
# relayAddr might be None if it's our parent, which is OK because
# the default message forwarding is to the parent. If it's not
# none, it should be in self._queues though!
self._fwdvia.add(sendAddr, relayAddr)
if hasattr(self, '_addressMgr'):
destAddr,msg = self._addressMgr.prepMessageSend(destAddr, msg)
if destAddr is None:
thesplog('Unexpected target inaccessibility for %s', msg,
level = logging.WARNING)
raise CannotPickleAddress(destAddr)
if msg is SendStatus.DeadTarget:
thesplog('Faking message "sent" because target is dead and recursion avoided.')
continue
if self.isMyAddress(destAddr):
if incoming_handler is None:
return ReceiveEnvelope(sendAddr, msg)
r = incoming_handler(ReceiveEnvelope(sendAddr, msg))
if not r:
return r # handler returned False, indicating run() should exit
else:
# Note: the following code has implicit knowledge of serialize() and xmit
putQValue = lambda relayer: (relayer, (sendAddr, destAddr, msg))
deadQValue = lambda relayer: (relayer, (sendAddr,
self._adminAddr,
DeadEnvelope(destAddr, msg)))
# Must forward this packet via a known forwarder or our parent.
send_dead = False
tgtQ = self._queues.find(destAddr)
if tgtQ:
sendArgs = putQValue(for_local_addr), True
if not tgtQ:
tgtA = self._fwdvia.find(destAddr)
if tgtA:
tgtQ = self._queues.find(tgtA)
sendArgs = putQValue(None),
else:
for each in self._deadaddrs:
if destAddr == each:
send_dead = True
if tgtQ:
try:
tgtQ.put(*sendArgs,
timeout=timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
continue
except Q.Full:
thesplog('Unable to send msg %s to dest %s; dead lettering',
msg, destAddr)
send_dead = True
if send_dead:
try:
(self._parentQ or self._adminQ).put(
deadQValue(for_local_addr if self._parentQ else None),
True,
timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
thesplog('Unable to send deadmsg %s to %s or admin; discarding',
msg, destAddr)
continue
# Not sure how to route this message yet. It
# could be a heretofore silent child of one of our
# children, it could be our parent (whose address
# we don't know), or it could be elsewhere in the
# tree.
#
# Try sending it to the parent first. If the
# parent can't determine the routing, it will be
# sent back down (relayAddr will be None in that
# case) and it must be sprayed out to all children
# in case the target lives somewhere beneath us.
# Note that _parentQ will be None for top-level
# actors, which send up to the Admin instead.
#
# As a special case, the external system is the
# parent of the admin, but the admin is the
# penultimate parent of all others, so this code
# must keep the admin and the parent from playing
# ping-pong with the message. But... the message
# might be directed to the external system, which
# is the parent of the Admin, so we need to check
# with it first.
# parentQ == None but adminQ good --> external
# parentQ and adminQ and myAddress == adminAddr --> Admin
# parentQ and adminQ and myAddress != adminADdr --> other Actor
if relayAddr:
# Send message up to the parent to see if the
# parent knows how to forward it
try:
(self._parentQ or self._adminQ).put(
putQValue(for_local_addr if self._parentQ else None),
True,
timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
thesplog('Unable to send dead msg %s to %s or admin; discarding',
msg, destAddr)
else:
# Sent by parent or we are an external, so this
# may be some grandchild not currently known.
# Do the worst case and just send this message
# to ALL immediate children, hoping it will
# get there via some path.
for A,AQ in self._queues.items():
if A not in [self._adminAddr, str(self._adminAddr)]:
# None means sent by Parent, so don't
# send BACK to parent if unknown
try:
AQ.put(putQValue(None),
True,
timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
pass
return None
def interrupt_run(self, signal_shutdown=False, check_children=False):
self._shutdownSignalled |= signal_shutdown
self._checkChildren |= check_children
# Do not put anything on the Queue if running in the context
# of a signal handler, because Queues are not signal-context
# safe. Instead, those will just have to depend on the short
# maximum Queue get wait time.
if not signal_shutdown and not check_children:
self._myInputQ.put_nowait('BuMP')
class LocalConventionState(object):
def __init__(self, myAddress, capabilities, sCBStats,
getConventionAddressFunc):
self._myAddress = myAddress
self._capabilities = capabilities
self._sCBStats = sCBStats
self._conventionMembers = AssocList() # key=Remote Admin Addr, value=ConventionMemberData
self._conventionNotificationHandlers = []
self._getConventionAddr = getConventionAddressFunc
self._conventionAddress = getConventionAddressFunc(capabilities)
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
self._has_been_activated = False
self._invited = False # entered convention as a result of an explicit invite
@property
def myAddress(self):
return self._myAddress
@property
def capabilities(self):
return self._capabilities
def updateStatusResponse(self, resp):
resp.setConventionLeaderAddress(self.conventionLeaderAddr)
resp.setConventionRegisterTime(self._conventionRegistration)
for each in self._conventionMembers.values():
resp.addConventioneer(each.remoteAddress, each.registryValid)
resp.setNotifyHandlers(self._conventionNotificationHandlers)
def active_in_convention(self):
# If this is the convention leader, it is automatically
# active, otherwise this convention member should have a
# convention leader and that leader should have an active
# entry in the _conventionMembers table (indicating it has
# updated this system with its information)
return bool(self.conventionLeaderAddr and
self._conventionMembers.find(self.conventionLeaderAddr))
@property
def conventionLeaderAddr(self):
return self._conventionAddress
def isConventionLeader(self):
# Might also be the leader if self.conventionLeaderAddr is None
return self.conventionLeaderAddr == self.myAddress
def capabilities_have_changed(self, new_capabilities):
self._capabilities = new_capabilities
return self.setup_convention()
def setup_convention(self, activation=False):
self._has_been_activated |= activation
rmsgs = []
# If not specified in capabilities, don't override any invites
# that may have been received.
self._conventionAddress = self._getConventionAddr(self.capabilities) or \
self._conventionAddress
leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \
if self.conventionLeaderAddr else True
if not self.isConventionLeader() and self.conventionLeaderAddr:
thesplog('Admin registering with Convention @ %s (%s)',
self.conventionLeaderAddr,
'first time' if leader_is_gone else 're-registering',
level=logging.INFO, primary=True)
rmsgs.append(
HysteresisSend(self.conventionLeaderAddr,
ConventionRegister(self.myAddress,
self.capabilities,
leader_is_gone),
onSuccess = self._setupConventionCBGood,
onError = self._setupConventionCBError))
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def _setupConventionCBGood(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registered')
if hasattr(self, '_conventionLeaderMissCount'):
delattr(self, '_conventionLeaderMissCount')
def _setupConventionCBError(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registration Failed')
if hasattr(self, '_conventionLeaderMissCount'):
self._conventionLeaderMissCount += 1
else:
self._conventionLeaderMissCount = 1
thesplog('Admin cannot register with convention @ %s (miss %d): %s',
finishedIntent.targetAddr,
self._conventionLeaderMissCount,
result, level=logging.WARNING, primary=True)
def got_convention_invite(self, sender):
self._conventionAddress = sender
self._invited = True
return self.setup_convention()
def got_convention_register(self, regmsg):
# Called when remote convention member has sent a ConventionRegister message
self._sCBStats.inc('Admin Handle Convention Registration')
if self._invited and not self.conventionLeaderAddr:
# Lost connection to an invitation-only convention.
# Cannot join again until another invitation is received.
return []
# Registrant may re-register if changing capabilities
rmsgs = []
registrant = regmsg.adminAddress
prereg = getattr(regmsg, 'preRegister', False) # getattr used; see definition
existing = self._conventionMembers.find(registrant)
thesplog('Got Convention %sregistration from %s (%s) (new? %s)',
'pre-' if prereg else '',
registrant,
'first time' if regmsg.firstTime else 're-registering',
not existing,
level=logging.INFO)
if registrant == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog('Convention registration from %s is an invalid address; ignoring.',
registrant,
level=logging.WARNING)
return rmsgs
existingPreReg = (
# existing.preRegOnly
# or existing.preRegistered
existing.permanentEntry
) if existing else False
notify = (not existing or existing.preRegOnly) and not prereg
if regmsg.firstTime or not existing:
if existing:
existing = None
notify = not prereg
rmsgs.extend(self._remote_system_cleanup(registrant))
newmember = ConventionMemberData(registrant,
regmsg.capabilities,
prereg)
if prereg or existingPreReg:
newmember.preRegistered = PreRegistration()
self._conventionMembers.add(registrant, newmember)
else:
existing.refresh(regmsg.capabilities, prereg or existingPreReg)
if not prereg:
existing.preRegOnly = False
if not self.isConventionLeader():
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
# Convention Members normally periodically initiate a
# membership message, to which the leader confirms by
# responding; if this was a pre-registration, that identifies
# this system as the "leader" for that remote. Also, if the
# remote sent this because it was a pre-registration leader,
# it doesn't yet have all the member information so the member
# should respond.
#if self.isConventionLeader() or prereg or regmsg.firstTime:
if prereg:
rmsgs.append(HysteresisCancel(registrant))
rmsgs.append(TransmitIntent(registrant, ConventionInvite()))
elif (self.isConventionLeader() or prereg or regmsg.firstTime or \
(existing and existing.permanentEntry)):
# If we are the Convention Leader, this would be the point to
# inform all other registrants of the new registrant. At
# present, there is no reciprocity here, so just update the
# new registrant with the leader's info.
rmsgs.append(
TransmitIntent(registrant,
ConventionRegister(self.myAddress,
self.capabilities)))
if notify:
rmsgs.extend(self._notifications_of(
ActorSystemConventionUpdate(registrant,
regmsg.capabilities,
True)))
return rmsgs
def _notifications_of(self, msg):
return [TransmitIntent(H, msg) for H in self._conventionNotificationHandlers]
def add_notification_handler(self, addr):
if addr not in self._conventionNotificationHandlers:
self._conventionNotificationHandlers.append(addr)
# Now update the registrant on the current state of all convention members
return [TransmitIntent(addr,
ActorSystemConventionUpdate(M.remoteAddress,
M.remoteCapabilities,
True))
for M in self._conventionMembers.values()
if not M.preRegOnly]
return []
def remove_notification_handler(self, addr):
self._conventionNotificationHandlers = [
H for H in self._conventionNotificationHandlers
if H != addr]
def got_convention_deregister(self, deregmsg):
self._sCBStats.inc('Admin Handle Convention De-registration')
remoteAdmin = deregmsg.adminAddress
if remoteAdmin == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog('Convention deregistration from %s is an invalid address; ignoring.',
remoteAdmin,
level=logging.WARNING)
rmsgs = []
if getattr(deregmsg, 'preRegistered', False): # see definition for getattr use
existing = self._conventionMembers.find(remoteAdmin)
if existing:
existing.preRegistered = None
rmsgs.append(TransmitIntent(remoteAdmin, ConventionDeRegister(self.myAddress)))
return rmsgs + self._remote_system_cleanup(remoteAdmin)
def got_system_shutdown(self):
return self.exit_convention()
def exit_convention(self):
self.invited = False
gen_ops = lambda addr: [HysteresisCancel(addr),
TransmitIntent(addr,
ConventionDeRegister(self.myAddress)),
]
terminate = lambda a: [ self._remote_system_cleanup(a), gen_ops(a) ][-1]
if self.conventionLeaderAddr and \
self.conventionLeaderAddr != self.myAddress:
thesplog('Admin de-registering with Convention @ %s',
str(self.conventionLeaderAddr),
level=logging.INFO, primary=True)
# Cache convention leader address because it might get reset by terminate()
claddr = self.conventionLeaderAddr
terminate(self.conventionLeaderAddr)
return gen_ops(claddr)
return join(fmap(terminate,
[M.remoteAddress
for M in self._conventionMembers.values()
if M.remoteAddress != self.myAddress]))
def check_convention(self):
rmsgs = []
if self._has_been_activated:
rmsgs = foldl(lambda x, y: x + y,
[self._check_preregistered_ping(member)
for member in self._conventionMembers.values()],
self._convention_leader_checks()
if self.isConventionLeader() or
not self.conventionLeaderAddr else
self._convention_member_checks())
if self._conventionRegistration.expired():
self._conventionRegistration = ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def _convention_leader_checks(self):
return foldl(lambda x, y: x + y,
[self._missed_checkin_remote_cleanup(R)
for R in [ member
for member in self._conventionMembers.values()
if member.registryValid.expired() ]],
[])
def _missed_checkin_remote_cleanup(self, remote_member):
thesplog('%s missed %d checkins (%s); assuming it has died',
str(remote_member),
CONVENTION_REGISTRATION_MISS_MAX,
str(remote_member.registryValid),
level=logging.WARNING, primary=True)
return self._remote_system_cleanup(remote_member.remoteAddress)
def _convention_member_checks(self):
rmsgs = []
# Re-register with the Convention if it's time
if self.conventionLeaderAddr and self._conventionRegistration.expired():
if getattr(self, '_conventionLeaderMissCount', 0) >= \
CONVENTION_REGISTRATION_MISS_MAX:
thesplog('Admin convention registration lost @ %s (miss %d)',
self.conventionLeaderAddr,
self._conventionLeaderMissCount,
level=logging.WARNING, primary=True)
rmsgs.extend(self._remote_system_cleanup(self.conventionLeaderAddr))
self._conventionLeaderMissCount = 0
else:
rmsgs.extend(self.setup_convention())
return rmsgs
def _check_preregistered_ping(self, member):
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
# If remote misses a checkin, re-extend the
# invitation. This also helps re-initiate a socket
# connection if a TxOnly socket has been lost.
member.preRegistered.pingValid = ExpirationTimer(
convention_reinvite_adjustment(
CONVENTION_RESTART_PERIOD
if member.registryValid.expired()
else CONVENTION_REREGISTRATION_PERIOD))
return [HysteresisSend(member.remoteAddress,
ConventionInvite(),
onSuccess = self._preRegQueryNotPending,
onError = self._preRegQueryNotPending)]
return []
def _preRegQueryNotPending(self, result, finishedIntent):
remoteAddr = finishedIntent.targetAddr
member = self._conventionMembers.find(remoteAddr)
if member and member.preRegistered:
member.preRegistered.pingPending = False
def _remote_system_cleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership. This is also called on
a First Time connection from the remote to discard any
previous connection information.
"""
thesplog('Convention cleanup or deregistration for %s (known? %s)',
registrant,
bool(self._conventionMembers.find(registrant)),
level=logging.INFO)
rmsgs = [LostRemote(registrant)]
cmr = self._conventionMembers.find(registrant)
if not cmr or cmr.preRegOnly:
return []
# Send exited notification to conventionNotificationHandler (if any)
for each in self._conventionNotificationHandlers:
rmsgs.append(
TransmitIntent(each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
if registrant == self.conventionLeaderAddr and self._invited:
self._conventionAddress = None
# Don't clear invited: once invited, that
# perpetually indicates this should be only a
# member and never a leader.
self._conventionMembers.rmv(registrant)
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpirationTimer(None)
cmr.preRegOnly = True
return rmsgs + [HysteresisCancel(registrant)]
def sentByRemoteAdmin(self, envelope):
for each in self._conventionMembers.values():
if envelope.sender == each.remoteAddress:
return True
return False
def convention_inattention_delay(self):
return self._conventionRegistration or \
ExpirationTimer(CONVENTION_REREGISTRATION_PERIOD
if self.active_in_convention() or
self.isConventionLeader() else
CONVENTION_RESTART_PERIOD)
def forward_pending_to_remote_system(self, childClass, envelope, sourceHash, acceptsCaps):
alreadyTried = getattr(envelope.message, 'alreadyTried', [])
remoteCandidates = [
K
for K in self._conventionMembers.values()
if not K.registryValid.expired()
and K.remoteAddress != envelope.sender # source Admin
and K.remoteAddress not in alreadyTried
and acceptsCaps(K.remoteCapabilities)]
if not remoteCandidates:
if self.isConventionLeader() or not self.conventionLeaderAddr:
raise NoCompatibleSystemForActor(
childClass,
'No known ActorSystems can handle a %s for %s',
childClass, envelope.message.forActor)
# Let the Convention Leader try to find an appropriate ActorSystem
bestC = self.conventionLeaderAddr
else:
# distribute equally amongst candidates
C = [(K.remoteAddress, len(K.hasRemoteActors))
for K in remoteCandidates]
bestC = foldl(lambda best,possible:
best if best[1] <= possible[1] else possible,
C)[0]
thesplog('Requesting creation of %s%s on remote admin %s',
envelope.message.actorClassName,
' (%s)'%sourceHash if sourceHash else '',
bestC)
if bestC not in alreadyTried:
# Don't send request to this remote again, it has already
# been tried. This would also be indicated by that system
# performing the add of self.myAddress as below, but if
# there is disagreement between the local and remote
# addresses, this addition will prevent continual
# bounceback.
alreadyTried.append(bestC)
if self.myAddress not in alreadyTried:
# Don't send request back to this actor system: it cannot
# handle it
alreadyTried.append(self.myAddress)
envelope.message.alreadyTried = alreadyTried
return [TransmitIntent(bestC, envelope.message)]
def send_to_all_members(self, message, exception_list=None):
return [HysteresisSend(M.remoteAddress, message)
for M in self._conventionMembers.values()
if M.remoteAddress not in (exception_list or [])]
class MultiprocessQueueTransport(asyncTransportBase, wakeupTransportBase):
"""A transport designed to use a multiprocess.Queue instance to send
and receive messages with other multiprocess Process actors.
There is one instance of this object in each Actor. This
object maintains a single input queue (used by its parent an
any children it creates) and a table of all known sub-Actor
addresses and their queues (being the most immediate child
Actor queue that moves the message closer to the child target,
or the Parent actor queue to which the message should be
forwarded if no child is identified).
"""
def __init__(self, initType, *args):
super(MultiprocessQueueTransport, self).__init__()
if isinstance(initType, ExternalInterfaceTransportInit):
# External process that's going to talk "in". There is no
# parent, and the child is the systemAdmin.
capabilities, logDefs, self._concontext = args
self._parentQ = None
NewQ = self._concontext.Queue if self._concontext else Queue
self._adminQ = NewQ(MAX_ADMIN_QUEUESIZE)
self._adminAddr = self.getAdminAddr(capabilities)
self._myQAddress = ActorAddress(QueueActorAddress('~'))
self._myInputQ = NewQ(MAX_ACTOR_QUEUESIZE)
elif isinstance(initType, MpQTEndpoint):
_addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr, ccon = initType.args
self._concontext = ccon
self._parentQ = parentQ
self._adminQ = adminQ
self._adminAddr = adminAddr
self._myQAddress = myAddr
self._myInputQ = myQueue
else:
thesplog('MultiprocessQueueTransport init of type %s unsupported!', str(initType),
level=logging.ERROR)
# _queues is a map of direct child ActorAddresses to Queue instance. Note
# that there will be multiple keys mapping to the same Queue
# instance because routing is only either to the Parent or to
# an immediate Child.
self._queues = AssocList() # addr -> queue
# _fwdvia represents routing for other than immediate parent
# or child (there may be multiple target addresses mapping to
# the same forward address.
self._fwdvia = AssocList() # targetAddress -> fwdViaAddress
self._nextSubInstance = 0
def protectedFileNumList(self):
return foldl(lambda a, b: a+[b._reader.fileno(), b._writer.fileno()],
[self._myInputQ, self._parentQ, self._adminQ] +
list(self._queues.values()), [])
def childResetFileNumList(self):
return foldl(lambda a, b: a+[b._reader.fileno(), b._writer.fileno()],
[self._parentQ] +
list(self._queues.values()), [])
@property
def myAddress(self): return self._myQAddress
@staticmethod
def getAddressFromString(addrspec):
# addrspec is assumed to be a valid address string
return ActorAddress(QueueActorAddress(addrspec))
@staticmethod
def getAdminAddr(capabilities):
return MultiprocessQueueTransport.getAddressFromString(
capabilities.get('Admin Address', 'ThespianQ'))
@staticmethod
def probeAdmin(addr):
"""Called to see if there might be an admin running already at the
specified addr. This is called from the systemBase, so
simple blocking operations are fine. This only needs to
check for a responder; higher level logic will verify that
it's actually an ActorAdmin suitable for use.
"""
# never reconnectable; Queue objects are only available from
# the constructor and cannot be synthesized or passed.
return False
def _updateStatusResponse(self, resp):
"Called to update a Thespian_SystemStatus or Thespian_ActorStatus with common information"
asyncTransportBase._updateStatusResponse(self, resp)
wakeupTransportBase._updateStatusResponse(self, resp)
def _nextSubAddress(self):
subAddrStr = self._myQAddress.addressDetails.subAddr(self._nextSubInstance)
self._nextSubInstance = self._nextSubInstance + 1
return ActorAddress(QueueActorAddress(subAddrStr))
def prepEndpoint(self, assignedLocalAddr, capabilities):
"""In the parent, prepare to establish a new communications endpoint
with a new Child Actor. The result of this call will be
passed to a created child process to use when initializing
the Transport object for that class; the result of this
call will also be kept by the parent to finalize the
communications after creation of the Child by calling
connectEndpoint() with this returned object.
"""
NewQ = self._concontext.Queue if self._concontext else Queue
if isinstance(assignedLocalAddr.addressDetails, ActorLocalAddress):
return MpQTEndpoint(assignedLocalAddr.addressDetails.addressInstanceNum,
self._nextSubAddress(),
NewQ(MAX_ACTOR_QUEUESIZE),
self._myInputQ, self._adminQ, self._adminAddr,
self._concontext)
return MpQTEndpoint(None,
assignedLocalAddr,
self._adminQ, self._myInputQ, self._adminQ,
self._adminAddr,
self._concontext)
def connectEndpoint(self, endPoint):
"""Called by the Parent after creating the Child to fully connect the
endpoint to the Child for ongoing communications."""
(_addrInst, childAddr, childQueue, _myQ,
_adminQ, _adminAddr, _concurrency_context) = endPoint.args
self._queues.add(childAddr, childQueue)
def deadAddress(self, addressManager, childAddr):
# Can no longer send to this Queue object. Delete the
# entry; this will cause forwarding of messages, although
# the addressManager is also aware of the dead address and
# will cause DeadEnvelope forwarding. Deleting here
# prevents hanging on queue full to dead children.
self._queues.rmv(childAddr)
deadfwd, okfwd = ([],[]) if False else \
partition(lambda i: i[0] == childAddr or i[1] == childAddr,
self._fwdvia.items())
if deadfwd:
self._fwdvia = AssocList()
for A,AQ in okfwd:
self._fwdvia.add(A,AQ)
super(MultiprocessQueueTransport, self).deadAddress(addressManager, childAddr)
def _runWithExpiry(self, incomingHandler):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
if incomingHandler == TransmitOnly or \
isinstance(incomingHandler, TransmitOnly):
# transmits are not queued/multistage in this transport, no waiting
return 0
self._aborting_run = False
while not self.run_time.expired() and not self._aborting_run:
try:
rcvd = self._myInputQ.get(True, self.run_time.remainingSeconds())
except Q.Empty:
# Probably a timeout, but let the while loop decide for sure
continue
if rcvd == 'BuMP':
return Thespian__UpdateWork()
relayAddr, (sendAddr, destAddr, msg) = rcvd
if not self._queues.find(sendAddr):
# We don't directly know about this sender, so
# remember what path this arrived on to know where to
# direct future messages for this sender.
if relayAddr and self._queues.find(relayAddr) and \
not self._fwdvia.find(sendAddr):
# relayAddr might be None if it's our parent, which is OK because
# the default message forwarding is to the parent. If it's not
# none, it should be in self._queues though!
self._fwdvia.add(sendAddr, relayAddr)
if hasattr(self, '_addressMgr'):
destAddr,msg = self._addressMgr.prepMessageSend(destAddr, msg)
if destAddr is None:
thesplog('Unexpected target inaccessibility for %s', msg,
level = logging.WARNING)
raise CannotPickleAddress(destAddr)
if msg is SendStatus.DeadTarget:
thesplog('Faking message "sent" because target is dead and recursion avoided.')
continue
if destAddr == self._myQAddress:
if incomingHandler is None:
return ReceiveEnvelope(sendAddr, msg)
if not incomingHandler(ReceiveEnvelope(sendAddr, msg)):
return # handler returned False, indicating run() should exit
else:
# Note: the following code has implicit knowledge of serialize() and xmit
putQValue = lambda relayer: (relayer, (sendAddr, destAddr, msg))
deadQValue = lambda relayer: (relayer, (sendAddr,
self._adminAddr,
DeadEnvelope(destAddr, msg)))
# Must forward this packet via a known forwarder or our parent.
tgtQ = self._queues.find(destAddr)
if tgtQ:
sendArgs = putQValue(self.myAddress), True
if not tgtQ:
tgtA = self._fwdvia.find(destAddr)
if tgtA:
tgtQ = self._queues.find(tgtA)
sendArgs = putQValue(None),
if tgtQ:
try:
tgtQ.put(*sendArgs,
timeout=timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
thesplog('Unable to send msg %s to dest %s; dead lettering',
msg, destAddr)
try:
(self._parentQ or self._adminQ).put(
deadQValue(self.myAddress if self._parentQ else None),
True,
timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
thesplog('Unable to send deadmsg %s to %s or admin; discarding',
msg, destAddr)
else:
# Not sure how to route this message yet. It
# could be a heretofore silent child of one of our
# children, it could be our parent (whose address
# we don't know), or it could be elsewhere in the
# tree.
#
# Try sending it to the parent first. If the
# parent can't determine the routing, it will be
# sent back down (relayAddr will be None in that
# case) and it must be sprayed out to all children
# in case the target lives somewhere beneath us.
# Note that _parentQ will be None for top-level
# actors, which send up to the Admin instead.
#
# As a special case, the external system is the
# parent of the admin, but the admin is the
# penultimate parent of all others, so this code
# must keep the admin and the parent from playing
# ping-pong with the message. But... the message
# might be directed to the external system, which
# is the parent of the Admin, so we need to check
# with it first.
# parentQ == None but adminQ good --> external
# parentQ and adminQ and myAddress == adminAddr --> Admin
# parentQ and adminQ and myAddress != adminADdr --> other Actor
if relayAddr:
# Send message up to the parent to see if the
# parent knows how to forward it
try:
(self._parentQ or self._adminQ).put(
putQValue(self.myAddress if self._parentQ else None),
True,
timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
thesplog('Unable to send dead msg %s to %s or admin; discarding',
msg, destAddr)
else:
# Sent by parent or we are an external, so this
# may be some grandchild not currently known.
# Do the worst case and just send this message
# to ALL immediate children, hoping it will
# get there via some path.
for A,AQ in self._queues.items():
if A not in [self._adminAddr, str(self._adminAddr)]:
# None means sent by Parent, so don't
# send BACK to parent if unknown
try:
AQ.put(putQValue(None),
True,
timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
pass
return None
def abort_run(self, drain=False):
# Queue transmits immediately, so no draining needed
self._aborting_run = True
def serializer(self, intent):
wrappedMsg = self._myQAddress, intent.targetAddr, intent.message
# For multiprocess Queues, the serialization (pickling) of the
# outbound message happens in a separate process. This is
# unfortunate because if the message is not pickle-able, the
# exception is thrown (and not handled) in the other process,
# and this process has no indication of the issue. The
# unfortunate solution is that pickling must be tried in the
# current process first to detect these errors (unfortunate
# because that means each message gets pickled twice,
# impacting performance).
discard = pickle.dumps(wrappedMsg)
return wrappedMsg
def interrupt_wait(self):
self._myInputQ.put_nowait('BuMP')
def _scheduleTransmitActual(self, transmitIntent):
try:
if transmitIntent.targetAddr == self.myAddress:
if transmitIntent.message:
self._myInputQ.put( (self._myQAddress, transmitIntent.serMsg), True,
timePeriodSeconds(transmitIntent.delay()))
else:
tgtQ = self._queues.find(transmitIntent.targetAddr)
if tgtQ:
tgtQ.put((self._myQAddress, transmitIntent.serMsg), True,
timePeriodSeconds(transmitIntent.delay()))
else:
# None means sent by parent, so don't send BACK to parent if unknown
topOrFromBelow = self._myQAddress if self._parentQ else None
(self._parentQ or self._adminQ).put(
(topOrFromBelow, transmitIntent.serMsg),
True,
timePeriodSeconds(transmitIntent.delay()))
transmitIntent.tx_done(SendStatus.Sent)
return
except Q.Full:
pass
transmitIntent.tx_done(SendStatus.DeadTarget if not isinstance(
transmitIntent._message,
(ChildActorExited, ActorExitRequest)) else SendStatus.Failed)
thesplog('Q.Full %s to %s result %s', transmitIntent._message, transmitIntent.targetAddr, transmitIntent.result)
class MultiprocessQueueTCore_Common(object):
def __init__(self, myQueue, parentQ, adminQ, adminAddr):
self._myInputQ = myQueue
self._parentQ = parentQ
self._adminQ = adminQ
self._adminAddr = adminAddr
# _queues is a map of direct child ActorAddresses to Queue instance. Note
# that there will be multiple keys mapping to the same Queue
# instance because routing is only either to the Parent or to
# an immediate Child.
self._queues = AssocList() # addr -> queue
# _fwdvia represents routing for other than immediate parent
# or child (there may be multiple target addresses mapping to
# the same forward address.
self._fwdvia = AssocList() # targetAddress -> fwdViaAddress
self._deadaddrs = []
# Signals can set these to true; they should be checked and
# reset by the main processing loop. There is a small window
# where they could be missed because signals are not queued,
# but this should handle the majority of situations. Note
# that the Queue object is NOT signal-safe, so don't try to
# queue signals that way.
self._checkChildren = False
self._shutdownSignalled = False
def mainLocalInputQueueEndpoint(self):
return self._myInputQ
def adminQueueEndpoint(self):
return self._adminQ
@property
def adminAddr(self):
return self._adminAddr
def protectedFileNumList(self):
return foldl(lambda a, b: a + [b._reader.fileno(
), b._writer.fileno()], [self._myInputQ, self._parentQ, self._adminQ] +
list(self._queues.values()), [])
def childResetFileNumList(self):
return foldl(lambda a, b: a + [b._reader.fileno(),
b._writer.fileno()],
[self._parentQ] + list(self._queues.values()), [])
def add_endpoint(self, child_addr, child_queue):
self._queues.add(child_addr, child_queue)
def set_address_to_dead(self, child_addr):
self._queues.rmv(child_addr)
self._fwdvia.rmv(child_addr)
self._fwdvia.rmv_value(child_addr)
self._deadaddrs.append(child_addr)
def abort_core_run(self):
self._aborting_run = Thespian__Run_Terminated()
def core_common_transmit(self, transmit_intent, from_addr):
try:
if self.isMyAddress(transmit_intent.targetAddr):
if transmit_intent.message:
self._myInputQ.put(
(from_addr, transmit_intent.serMsg), True,
timePeriodSeconds(transmit_intent.delay()))
else:
tgtQ = self._queues.find(transmit_intent.targetAddr)
if tgtQ:
tgtQ.put((from_addr, transmit_intent.serMsg), True,
timePeriodSeconds(transmit_intent.delay()))
else:
# None means sent by parent, so don't send BACK to parent if unknown
topOrFromBelow = from_addr if self._parentQ else None
(self._parentQ or self._adminQ).put(
(topOrFromBelow, transmit_intent.serMsg), True,
timePeriodSeconds(transmit_intent.delay()))
transmit_intent.tx_done(SendStatus.Sent)
return
except Q.Full:
pass
transmit_intent.tx_done(
SendStatus.DeadTarget if not isinstance(transmit_intent._message, (
ChildActorExited, ActorExitRequest)) else SendStatus.Failed)
def core_common_receive(self, incoming_handler, local_routing_addr,
run_time_f):
"""Core scheduling method; called by the current Actor process when
idle to await new messages (or to do background
processing).
"""
if incoming_handler == TransmitOnly or \
isinstance(incoming_handler, TransmitOnly):
# transmits are not queued/multistage in this transport, no waiting
return local_routing_addr, 0
self._aborting_run = None
while self._aborting_run is None:
ct = currentTime()
if run_time_f().view(ct).expired():
break
try:
# Unfortunately, the Queue object is not signal-safe,
# so a frequent wakeup is needed to check
# _checkChildren and _shutdownSignalled.
rcvd = self._myInputQ.get(
True,
min(
run_time_f().view(ct).remainingSeconds()
or QUEUE_CHECK_PERIOD, QUEUE_CHECK_PERIOD))
except Q.Empty:
if not self._checkChildren and not self._shutdownSignalled:
# Probably a timeout, but let the while loop decide for sure
continue
rcvd = 'BuMP'
if rcvd == 'BuMP':
relayAddr = sendAddr = destAddr = local_routing_addr
if self._checkChildren:
self._checkChildren = False
msg = ChildMayHaveDied()
elif self._shutdownSignalled:
self._shutdownSignalled = False
msg = ActorExitRequest()
else:
return local_routing_addr, Thespian__UpdateWork()
else:
relayAddr, (sendAddr, destAddr, msg) = rcvd
if not self._queues.find(sendAddr):
# We don't directly know about this sender, so
# remember what path this arrived on to know where to
# direct future messages for this sender.
if relayAddr and self._queues.find(relayAddr) and \
not self._fwdvia.find(sendAddr):
# relayAddr might be None if it's our parent, which is OK because
# the default message forwarding is to the parent. If it's not
# none, it should be in self._queues though!
self._fwdvia.add(sendAddr, relayAddr)
if hasattr(self, '_addressMgr'):
destAddr, msg = self._addressMgr.prepMessageSend(destAddr, msg)
if destAddr is None:
thesplog('Unexpected target inaccessibility for %s',
msg,
level=logging.WARNING)
raise CannotPickleAddress(destAddr)
if msg is SendStatus.DeadTarget:
thesplog(
'Faking message "sent" because target is dead and recursion avoided.'
)
continue
if self.isMyAddress(destAddr):
if isinstance(incoming_handler,
ReturnTargetAddressWithEnvelope):
return destAddr, ReceiveEnvelope(sendAddr, msg)
if incoming_handler is None:
return destAddr, ReceiveEnvelope(sendAddr, msg)
r = Thespian__Run_HandlerResult(
incoming_handler(ReceiveEnvelope(sendAddr, msg)))
if not r:
# handler returned False-ish, indicating run() should exit
return destAddr, r
else:
# Note: the following code has implicit knowledge of serialize() and xmit
putQValue = lambda relayer: (relayer,
(sendAddr, destAddr, msg))
deadQValue = lambda relayer: (relayer,
(sendAddr, self._adminAddr,
DeadEnvelope(destAddr, msg)))
# Must forward this packet via a known forwarder or our parent.
send_dead = False
tgtQ = self._queues.find(destAddr)
if tgtQ:
sendArgs = putQValue(local_routing_addr), True
if not tgtQ:
tgtA = self._fwdvia.find(destAddr)
if tgtA:
tgtQ = self._queues.find(tgtA)
sendArgs = putQValue(None),
else:
for each in self._deadaddrs:
if destAddr == each:
send_dead = True
if tgtQ:
try:
tgtQ.put(*sendArgs,
timeout=timePeriodSeconds(
MAX_QUEUE_TRANSMIT_PERIOD))
continue
except Q.Full:
thesplog(
'Unable to send msg %s to dest %s; dead lettering',
msg, destAddr)
send_dead = True
if send_dead:
try:
(self._parentQ or self._adminQ).put(
deadQValue(
local_routing_addr if self._parentQ else None),
True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
thesplog(
'Unable to send deadmsg %s to %s or admin; discarding',
msg, destAddr)
continue
# Not sure how to route this message yet. It
# could be a heretofore silent child of one of our
# children, it could be our parent (whose address
# we don't know), or it could be elsewhere in the
# tree.
#
# Try sending it to the parent first. If the
# parent can't determine the routing, it will be
# sent back down (relayAddr will be None in that
# case) and it must be sprayed out to all children
# in case the target lives somewhere beneath us.
# Note that _parentQ will be None for top-level
# actors, which send up to the Admin instead.
#
# As a special case, the external system is the
# parent of the admin, but the admin is the
# penultimate parent of all others, so this code
# must keep the admin and the parent from playing
# ping-pong with the message. But... the message
# might be directed to the external system, which
# is the parent of the Admin, so we need to check
# with it first.
# parentQ == None but adminQ good --> external
# parentQ and adminQ and myAddress == adminAddr --> Admin
# parentQ and adminQ and myAddress != adminADdr --> other Actor
if relayAddr:
# Send message up to the parent to see if the
# parent knows how to forward it
try:
(self._parentQ or self._adminQ).put(
putQValue(
local_routing_addr if self._parentQ else None),
True, timePeriodSeconds(MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
thesplog(
'Unable to send dead msg %s to %s or admin; discarding',
msg, destAddr)
else:
# Sent by parent or we are an external, so this
# may be some grandchild not currently known.
# Do the worst case and just send this message
# to ALL immediate children, hoping it will
# get there via some path.
for A, AQ in self._queues.items():
if A not in [self._adminAddr, str(self._adminAddr)]:
# None means sent by Parent, so don't
# send BACK to parent if unknown
try:
AQ.put(
putQValue(None), True,
timePeriodSeconds(
MAX_QUEUE_TRANSMIT_PERIOD))
except Q.Full:
pass
if self._aborting_run is not None:
return local_routing_addr, self._aborting_run
return local_routing_addr, Thespian__Run_Expired()
def interrupt_run(self, signal_shutdown=False, check_children=False):
self._shutdownSignalled |= signal_shutdown
self._checkChildren |= check_children
# Do not put anything on the Queue if running in the context
# of a signal handler, because Queues are not signal-context
# safe. Instead, those will just have to depend on the short
# maximum Queue get wait time.
if not signal_shutdown and not check_children:
try:
self._myInputQ.put_nowait('BuMP')
except Q.Full:
# if the queue is full, it should be reading something
# off soon which will accomplish the same interrupt
# effect, so nothing else needs to be done here.
pass
class LocalConventionState(object):
def __init__(self, myAddress, capabilities, sCBStats,
getConventionAddressFunc):
self._myAddress = myAddress
self._capabilities = capabilities
self._sCBStats = sCBStats
self._conventionMembers = AssocList(
) # key=Remote Admin Addr, value=ConventionMemberData
self._conventionNotificationHandlers = []
self._getConventionAddr = getConventionAddressFunc
self._conventionAddress = getConventionAddressFunc(capabilities)
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
self._has_been_activated = False
@property
def myAddress(self):
return self._myAddress
@property
def capabilities(self):
return self._capabilities
def updateStatusResponse(self, resp):
resp.setConventionLeaderAddress(self.conventionLeaderAddr)
resp.setConventionRegisterTime(self._conventionRegistration)
for each in self._conventionMembers.values():
resp.addConventioneer(each.remoteAddress, each.registryValid)
resp.setNotifyHandlers(self._conventionNotificationHandlers)
def active_in_convention(self):
return self.conventionLeaderAddr and \
not self._conventionMembers.find(self.conventionLeaderAddr)
@property
def conventionLeaderAddr(self):
return self._conventionAddress
def isConventionLeader(self):
return self.conventionLeaderAddr == self.myAddress
def capabilities_have_changed(self, new_capabilities):
self._capabilities = new_capabilities
return self.setup_convention()
def setup_convention(self, activation=False):
self._has_been_activated |= activation
rmsgs = []
self._conventionAddress = self._getConventionAddr(self.capabilities)
leader_is_gone = (
self._conventionMembers.find(self.conventionLeaderAddr) is None
) if self.conventionLeaderAddr else True
if not self.isConventionLeader() and self.conventionLeaderAddr:
thesplog('Admin registering with Convention @ %s (%s)',
self.conventionLeaderAddr,
'first time' if leader_is_gone else 're-registering',
level=logging.INFO,
primary=True)
rmsgs.append(
HysteresisSend(self.conventionLeaderAddr,
ConventionRegister(self.myAddress,
self.capabilities,
leader_is_gone),
onSuccess=self._setupConventionCBGood,
onError=self._setupConventionCBError))
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def _setupConventionCBGood(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registered')
if hasattr(self, '_conventionLeaderMissCount'):
delattr(self, '_conventionLeaderMissCount')
def _setupConventionCBError(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registration Failed')
if hasattr(self, '_conventionLeaderMissCount'):
self._conventionLeaderMissCount += 1
else:
self._conventionLeaderMissCount = 1
if self._conventionLeaderMissCount < CONVENTION_REGISTRATION_MISS_MAX:
thesplog(
'Admin cannot register with convention @ %s (miss %d): %s',
finishedIntent.targetAddr,
self._conventionLeaderMissCount,
result,
level=logging.WARNING,
primary=True)
else:
thesplog('Admin convention registration lost @ %s (miss %d): %s',
finishedIntent.targetAddr,
self._conventionLeaderMissCount,
result,
level=logging.ERROR,
primary=True)
def got_convention_invite(self, sender):
self._conventionAddress = sender
return self.setup_convention()
def got_convention_register(self, regmsg):
self._sCBStats.inc('Admin Handle Convention Registration')
# Registrant may re-register if changing capabilities
rmsgs = []
registrant = regmsg.adminAddress
prereg = getattr(regmsg, 'preRegister',
False) # getattr used; see definition
existing = self._conventionMembers.find(registrant)
thesplog('Got Convention %sregistration from %s (%s) (new? %s)',
'pre-' if prereg else '',
registrant,
'first time' if regmsg.firstTime else 're-registering',
not existing,
level=logging.INFO)
if registrant == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog(
'Convention registration from %s is an invalid address; ignoring.',
registrant,
level=logging.WARNING)
return rmsgs
#if prereg or (regmsg.firstTime and not prereg):
if regmsg.firstTime or (prereg and not existing):
# erase knowledge of actors associated with potential
# former instance of this system
existing = False
rmsgs.extend(self._remote_system_cleanup(registrant))
if existing:
existingPreReg = existing.preRegOnly
existing.refresh(regmsg.capabilities, prereg)
self._conventionMembers.add(registrant, existing)
else:
newmember = ConventionMemberData(registrant, regmsg.capabilities,
prereg)
if prereg:
# Need to attempt registration immediately
newmember.preRegistered = PreRegistration()
self._conventionMembers.add(registrant, newmember)
# Convention Members normally periodically initiate a
# membership message, to which the leader confirms by
# responding; if this was a pre-registration, that identifies
# this system as the "leader" for that remote. Also, if the
# remote sent this because it was a pre-registration leader,
# it doesn't yet have all the member information so the member
# should respond.
#if self.isConventionLeader() or prereg or regmsg.firstTime:
if self.isConventionLeader() or prereg or regmsg.firstTime or \
(existing and existing.preRegistered):
if not existing or not existing.preRegOnly: # or prereg:
first = prereg
# If we are the Convention Leader, this would be the point to
# inform all other registrants of the new registrant. At
# present, there is no reciprocity here, so just update the
# new registrant with the leader's info.
rmsgs.append(
TransmitIntent(
registrant,
ConventionRegister(self.myAddress, self.capabilities,
first)))
if (existing and existingPreReg and not existing.preRegOnly) or \
(not existing and newmember and not prereg):
rmsgs.extend(
self._notifications_of(
ActorSystemConventionUpdate(registrant,
regmsg.capabilities, True)))
return rmsgs
def _notifications_of(self, msg):
return [
TransmitIntent(H, msg)
for H in self._conventionNotificationHandlers
]
def add_notification_handler(self, addr):
if addr not in self._conventionNotificationHandlers:
self._conventionNotificationHandlers.append(addr)
# Now update the registrant on the current state of all convention members
return [
TransmitIntent(
addr,
ActorSystemConventionUpdate(M.remoteAddress,
M.remoteCapabilities, True))
for M in self._conventionMembers.values()
]
return []
def remove_notification_handler(self, addr):
self._conventionNotificationHandlers = [
H for H in self._conventionNotificationHandlers if H != addr
]
def got_convention_deregister(self, deregmsg):
self._sCBStats.inc('Admin Handle Convention De-registration')
remoteAdmin = deregmsg.adminAddress
if remoteAdmin == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog(
'Convention deregistration from %s is an invalid address; ignoring.',
remoteAdmin,
level=logging.WARNING)
if getattr(deregmsg, 'preRegistered',
False): # see definition for getattr use
existing = self._conventionMembers.find(remoteAdmin)
if existing:
existing.preRegistered = None
return self._remote_system_cleanup(remoteAdmin)
def got_system_shutdown(self):
gen_ops = lambda addr: [
HysteresisCancel(addr),
TransmitIntent(addr, ConventionDeRegister(self.myAddress)),
]
if self.conventionLeaderAddr and \
self.conventionLeaderAddr != self.myAddress:
thesplog('Admin de-registering with Convention @ %s',
str(self.conventionLeaderAddr),
level=logging.INFO,
primary=True)
return gen_ops(self.conventionLeaderAddr)
return join(
fmap(gen_ops, [
M.remoteAddress for M in self._conventionMembers.values()
if M.remoteAddress != self.myAddress
]))
def check_convention(self):
rmsgs = []
if not self._has_been_activated:
return rmsgs
if self.isConventionLeader() or not self.conventionLeaderAddr:
missing = []
for each in self._conventionMembers.values():
if each.registryValid.expired():
missing.append(each)
for each in missing:
thesplog('%s missed %d checkins (%s); assuming it has died',
str(each),
CONVENTION_REGISTRATION_MISS_MAX,
str(each.registryValid),
level=logging.WARNING,
primary=True)
rmsgs.extend(self._remote_system_cleanup(each.remoteAddress))
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
else:
# Re-register with the Convention if it's time
if self.conventionLeaderAddr and self._conventionRegistration.expired(
):
rmsgs.extend(self.setup_convention())
for member in self._conventionMembers.values():
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
member.preRegistered.pingValid = ExpiryTime(
CONVENTION_RESTART_PERIOD if member.registryValid.expired(
) else CONVENTION_REREGISTRATION_PERIOD)
rmsgs.append(
HysteresisSend(member.remoteAddress,
ConventionInvite(),
onSuccess=self._preRegQueryNotPending,
onError=self._preRegQueryNotPending))
return rmsgs
def _preRegQueryNotPending(self, result, finishedIntent):
remoteAddr = finishedIntent.targetAddr
member = self._conventionMembers.find(remoteAddr)
if member and member.preRegistered:
member.preRegistered.pingPending = False
def _remote_system_cleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership. This is also called on
a First Time connection from the remote to discard any
previous connection information.
"""
thesplog('Convention cleanup or deregistration for %s (known? %s)',
registrant,
bool(self._conventionMembers.find(registrant)),
level=logging.INFO)
rmsgs = [LostRemote(registrant)]
cmr = self._conventionMembers.find(registrant)
if cmr:
# Send exited notification to conventionNotificationHandler (if any)
for each in self._conventionNotificationHandlers:
rmsgs.append(
TransmitIntent(
each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
self._conventionMembers.rmv(registrant)
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpiryTime(None)
return rmsgs + [HysteresisCancel(registrant)]
def sentByRemoteAdmin(self, envelope):
for each in self._conventionMembers.values():
if envelope.sender == each.remoteAddress:
return True
return False
def convention_inattention_delay(self):
return self._conventionRegistration or \
ExpiryTime(CONVENTION_REREGISTRATION_PERIOD
if self.active_in_convention() or
self.isConventionLeader() else
CONVENTION_RESTART_PERIOD)
def forward_pending_to_remote_system(self, childClass, envelope,
sourceHash, acceptsCaps):
alreadyTried = getattr(envelope.message, 'alreadyTried', [])
remoteCandidates = [
K for K in self._conventionMembers.values()
if not K.registryValid.expired()
and K.remoteAddress != envelope.sender # source Admin
and K.remoteAddress not in alreadyTried
and acceptsCaps(K.remoteCapabilities)
]
if not remoteCandidates:
if self.isConventionLeader() or not self.conventionLeaderAddr:
raise NoCompatibleSystemForActor(
childClass, 'No known ActorSystems can handle a %s for %s',
childClass, envelope.message.forActor)
# Let the Convention Leader try to find an appropriate ActorSystem
bestC = self.conventionLeaderAddr
else:
# distribute equally amongst candidates
C = [(K.remoteAddress, len(K.hasRemoteActors))
for K in remoteCandidates]
bestC = foldl(
lambda best, possible: best
if best[1] <= possible[1] else possible, C)[0]
thesplog('Requesting creation of %s%s on remote admin %s',
envelope.message.actorClassName,
' (%s)' % sourceHash if sourceHash else '', bestC)
if bestC not in alreadyTried:
# Don't send request to this remote again, it has already
# been tried. This would also be indicated by that system
# performing the add of self.myAddress as below, but if
# there is disagreement between the local and remote
# addresses, this addition will prevent continual
# bounceback.
alreadyTried.append(bestC)
if self.myAddress not in alreadyTried:
# Don't send request back to this actor system: it cannot
# handle it
alreadyTried.append(self.myAddress)
envelope.message.alreadyTried = alreadyTried
return [TransmitIntent(bestC, envelope.message)]
def send_to_all_members(self, message, exception_list=None):
return [
HysteresisSend(M.remoteAddress, message)
for M in self._conventionMembers.values()
if M.remoteAddress not in (exception_list or [])
]
class ConventioneerAdmin(GlobalNamesAdmin):
"""Extends the AdminCore+GlobalNamesAdmin with ActorSystem Convention
functionality to support multi-host configurations.
"""
def __init__(self, *args, **kw):
super(ConventioneerAdmin, self).__init__(*args, **kw)
self._conventionMembers = AssocList(
) # key=Remote Admin Addr, value=ConventionMemberData
self._conventionRegistration = ExpiryTime(timedelta(seconds=0))
self._conventionNotificationHandlers = []
self._conventionAddress = None # Not a member; still could be a leader
self._pendingSources = {
} # key = sourceHash, value is array of PendingActor requests
self._hysteresisSender = HysteresisDelaySender(self._send_intent)
def _updateStatusResponse(self, resp):
resp.setConventionLeaderAddress(self._conventionAddress)
resp.setConventionRegisterTime(self._conventionRegistration)
for each in self._conventionMembers.values():
resp.addConventioneer(each.remoteAddress, each.registryValid)
resp.setNotifyHandlers(self._conventionNotificationHandlers)
super(ConventioneerAdmin, self)._updateStatusResponse(resp)
def _activate(self):
self.setupConvention()
@property
def _conventionLost(self):
"True if this system was part of a convention but are no longer"
return getattr(self, '_conventionLeaderIsGone', False)
def setupConvention(self):
if self.isShuttingDown(): return
if not self._conventionAddress:
gCA = getattr(self.transport, 'getConventionAddress',
lambda c: None)
self._conventionAddress = gCA(self.capabilities)
if self._conventionAddress == self.myAddress:
self._conventionAddress = None
if self._conventionAddress:
thesplog('Admin registering with Convention @ %s (%s)',
self._conventionAddress,
'first time' if getattr(self, '_conventionLeaderIsGone',
True) else 're-registering',
level=logging.INFO,
primary=True)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(self._conventionAddress,
ConventionRegister(
self.myAddress, self.capabilities,
getattr(self, '_conventionLeaderIsGone',
True)),
onSuccess=self._setupConventionCBGood,
onError=self._setupConventionCBError))
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
def _setupConventionCBGood(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registered')
self._conventionLeaderIsGone = False
if hasattr(self, '_conventionLeaderMissCount'):
delattr(self, '_conventionLeaderMissCount')
if getattr(self, 'asLogger', None):
thesplog('Setting log aggregator of %s to %s', self.asLogger,
self._conventionAddress)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.asLogger,
LogAggregator(self._conventionAddress)))
def _setupConventionCBError(self, result, finishedIntent):
self._sCBStats.inc('Admin Convention Registration Failed')
if hasattr(self, '_conventionLeaderMissCount'):
self._conventionLeaderMissCount += 1
else:
self._conventionLeaderMissCount = 1
if self._conventionLeaderMissCount < CONVENTION_REGISTRATION_MISS_MAX:
thesplog(
'Admin cannot register with convention @ %s (miss %d): %s',
finishedIntent.targetAddr,
self._conventionLeaderMissCount,
result,
level=logging.WARNING,
primary=True)
else:
thesplog('Admin convention registration lost @ %s (miss %d): %s',
finishedIntent.targetAddr,
self._conventionLeaderMissCount,
result,
level=logging.ERROR,
primary=True)
if hasattr(self, '_conventionLeaderIsGone'):
self._conventionLeaderIsGone = True
def isConventionLeader(self):
return self._conventionAddress is None or self._conventionAddress == self.myAddress
def h_ConventionInvite(self, envelope):
self._conventionAddress = envelope.sender
self.setupConvention()
def h_ConventionRegister(self, envelope):
if self.isShuttingDown(): return
self._sCBStats.inc('Admin Handle Convention Registration')
# Registrant may re-register if changing capabilities
registrant = envelope.message.adminAddress
thesplog(
'Got Convention registration from %s (%s) (new? %s)',
registrant,
'first time' if envelope.message.firstTime else 're-registering',
not self._conventionMembers.find(registrant),
level=logging.INFO)
if registrant == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog(
'Convention registration from %s is an invalid address; ignoring.',
registrant,
level=logging.WARNING)
return True
if envelope.message.firstTime:
# erase knowledge of actors associated with potential
# former instance of this system
self._remoteSystemCleanup(registrant)
if registrant == self._conventionAddress:
self._conventionLeaderIsGone = False
existing = self._conventionMembers.find(registrant)
if existing:
existing.refresh(envelope.message.capabilities)
else:
newmember = ConventionMemberData(registrant,
envelope.message.capabilities)
if getattr(envelope.message, 'preRegister',
False): # getattr used; see definition
newmember.preRegistered = PreRegistration(
) # will attempt registration immediately
self._conventionMembers.add(registrant, newmember)
if self.isConventionLeader():
if not existing:
for each in self._conventionNotificationHandlers:
self._send_intent(
TransmitIntent(
each,
ActorSystemConventionUpdate(
registrant, envelope.message.capabilities,
True))) # errors ignored
# If we are the Convention Leader, this would be the point to
# inform all other registrants of the new registrant. At
# present, there is no reciprocity here, so just update the
# new registrant with the leader's info.
self._send_intent(
TransmitIntent(
registrant,
ConventionRegister(
self.myAddress,
self.capabilities,
# first time in, then must be first time out
envelope.message.firstTime)))
return True
def h_ConventionDeRegister(self, envelope):
self._sCBStats.inc('Admin Handle Convention De-registration')
remoteAdmin = envelope.message.adminAddress
if remoteAdmin == self.myAddress:
# Either remote failed getting an external address and is
# using 127.0.0.1 or else this is a malicious attempt to
# make us talk to ourselves. Ignore it.
thesplog(
'Convention deregistration from %s is an invalid address; ignoring.',
remoteAdmin,
level=logging.WARNING)
return True
if getattr(envelope.message, 'preRegistered',
False): # see definition for getattr use
existing = self._conventionMembers.find(remoteAdmin)
if existing:
existing.preRegistered = None
self._remoteSystemCleanup(remoteAdmin)
return True
def h_SystemShutdown(self, envelope):
if self._conventionAddress:
thesplog('Admin de-registering with Convention @ %s',
str(self._conventionAddress),
level=logging.INFO,
primary=True)
self._hysteresisSender.cancelSends(self._conventionAddress)
self._send_intent(
TransmitIntent(self._conventionAddress,
ConventionDeRegister(self.myAddress)))
else:
for each in self._conventionMembers.values():
rmtaddr = each.remoteAddress
self._hysteresisSender.cancelSends(rmtaddr)
self._send_intent(
TransmitIntent(rmtaddr,
ConventionDeRegister(self.myAddress)))
return super(ConventioneerAdmin, self).h_SystemShutdown(envelope)
def _remoteSystemCleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership.
"""
thesplog('Convention cleanup or deregistration for %s (new? %s)',
registrant,
not self._conventionMembers.find(registrant),
level=logging.INFO)
if hasattr(self.transport, 'lostRemote'):
self.transport.lostRemote(registrant)
cmr = self._conventionMembers.find(registrant)
if cmr:
# Send exited notification to conventionNotificationHandler (if any)
if self.isConventionLeader():
for each in self._conventionNotificationHandlers:
self._send_intent(
TransmitIntent(
each,
ActorSystemConventionUpdate(
cmr.remoteAddress, cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
self._send_intent(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
self._conventionMembers.rmv(registrant)
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpiryTime(None)
if registrant == self._conventionAddress:
# Convention Leader has exited. Do NOT set
# conventionAddress to None. It might speed up shutdown
# of this ActorSystem because it won't try to de-register
# from the convention leader, but if the convention leader
# reappears there will be nothing to get this ActorSystem
# re-registered with the convention.
self._conventionLeaderIsGone = True
self._hysteresisSender.cancelSends(registrant)
def _checkConvention(self):
if self.isConventionLeader():
missing = []
for each in self._conventionMembers.values():
if each.registryValid.expired():
missing.append(each)
for each in missing:
thesplog('%s missed %d checkins (%s); assuming it has died',
str(each),
CONVENTION_REGISTRATION_MISS_MAX,
str(each.registryValid),
level=logging.WARNING,
primary=True)
self._remoteSystemCleanup(each.remoteAddress)
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
else:
# Re-register with the Convention if it's time
if self._conventionAddress and self._conventionRegistration.expired(
):
self.setupConvention()
for member in self._conventionMembers.values():
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
member.preRegistered.pingValid = ExpiryTime(
CONVENTION_RESTART_PERIOD if member.registryValid.expired(
) else CONVENTION_REREGISTRATION_PERIOD)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(member.remoteAddress,
ConventionInvite(),
onSuccess=self._preRegQueryNotPending,
onError=self._preRegQueryNotPending))
def _preRegQueryNotPending(self, result, finishedIntent):
remoteAddr = finishedIntent.targetAddr
member = self._conventionMembers.find(remoteAddr)
if member and member.preRegistered:
member.preRegistered.pingPending = False
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
try:
while not self.isShuttingDown():
delay = min(self._conventionRegistration or \
ExpiryTime(CONVENTION_RESTART_PERIOD
if self._conventionLost and not self.isConventionLeader() else
CONVENTION_REREGISTRATION_PERIOD),
ExpiryTime(None) if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay
)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
self.transport.run(self.handleIncoming, delay.remaining())
self._checkConvention()
self._hysteresisSender.checkSends()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
# ---- Source Hash Transfers --------------------------------------------------
def h_SourceHashTransferRequest(self, envelope):
sourceHash = envelope.message.sourceHash
src = self._sources.get(sourceHash, None)
self._send_intent(
TransmitIntent(
envelope.sender,
SourceHashTransferReply(sourceHash, src and src.zipsrc, src
and src.srcInfo)))
return True
def h_SourceHashTransferReply(self, envelope):
sourceHash = envelope.message.sourceHash
pending = self._pendingSources[sourceHash]
del self._pendingSources[sourceHash]
if envelope.message.isValid():
self._sources[sourceHash] = ValidSource(
sourceHash, envelope.message.sourceData,
getattr(envelope.message, 'sourceInfo', None))
for each in pending:
self.h_PendingActor(each)
else:
for each in pending:
self._sendPendingActorResponse(
each,
None,
errorCode=PendingActorResponse.ERROR_Invalid_SourceHash)
return True
def h_ValidateSource(self, envelope):
if not envelope.message.sourceData and envelope.sender != self._conventionAddress:
# Propagate source unload requests to all convention members
for each in self._conventionMembers.values():
# Do not propagate if this is where the notification
# came from; prevents indefinite bouncing of this
# message as long as the convention structure is a
# DAG.
if each.remoteAddress != envelope.sender:
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(each.remoteAddress, envelope.message))
super(ConventioneerAdmin, self).h_ValidateSource(envelope)
return False # might have sent with hysteresis, so break out to local _run
def _sentByRemoteAdmin(self, envelope):
for each in self._conventionMembers.values():
if envelope.sender == each.remoteAddress:
return True
return False
def _acceptsRemoteLoadedSourcesFrom(self, pendingActorEnvelope):
allowed = self.capabilities.get('AllowRemoteActorSources', 'yes')
return allowed.lower() == 'yes' or \
(allowed == 'LeaderOnly' and
pendingActorEnvelope.sender == self._conventionAddress)
# ---- Remote Actor interactions ----------------------------------------------
def h_PendingActor(self, envelope):
sourceHash = envelope.message.sourceHash
childRequirements = envelope.message.targetActorReq
thesplog('Pending Actor request received for %s%s reqs %s from %s',
envelope.message.actorClassName,
' (%s)' % sourceHash if sourceHash else '', childRequirements,
envelope.sender)
# If this request was forwarded by a remote Admin and the
# sourceHash is not known locally, request it from the sending
# remote Admin
if sourceHash and \
sourceHash not in self._sources and \
self._sentByRemoteAdmin(envelope) and \
self._acceptsRemoteLoadedSourcesFrom(envelope):
requestedAlready = self._pendingSources.get(sourceHash, False)
self._pendingSources.setdefault(sourceHash, []).append(envelope)
if not requestedAlready:
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(envelope.sender,
SourceHashTransferRequest(sourceHash)))
return False # sent with hysteresis, so break out to local _run
return True
# If the requested ActorClass is compatible with this
# ActorSystem, attempt to start it, otherwise forward the
# request to any known compatible ActorSystem.
try:
childClass = actualActorClass(
envelope.message.actorClassName,
partial(loadModuleFromHashSource, sourceHash, self._sources)
if sourceHash else None)
acceptsCaps = lambda caps: checkActorCapabilities(
childClass, caps, childRequirements)
if not acceptsCaps(self.capabilities):
if envelope.message.forActor is None:
# Request from external; use sender address
envelope.message.forActor = envelope.sender
remoteCandidates = [
K for K in self._conventionMembers.values()
if not K.registryValid.expired()
and K.remoteAddress != envelope.sender # source Admin
and K.remoteAddress not in getattr(envelope.message,
'alreadyTried', [])
and acceptsCaps(K.remoteCapabilities)
]
if not remoteCandidates:
if self.isConventionLeader():
thesplog(
'No known ActorSystems can handle a %s for %s',
childClass,
envelope.message.forActor,
level=logging.WARNING,
primary=True)
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.
ERROR_No_Compatible_ActorSystem)
return True
# Let the Convention Leader try to find an appropriate ActorSystem
bestC = self._conventionAddress
else:
# distribute equally amongst candidates
C = [(K.remoteAddress, len(K.hasRemoteActors))
for K in remoteCandidates]
bestC = foldl(
lambda best, possible: best
if best[1] <= possible[1] else possible, C)[0]
thesplog('Requesting creation of %s%s on remote admin %s',
envelope.message.actorClassName,
' (%s)' % sourceHash if sourceHash else '', bestC)
envelope.message.alreadyTried.append(self.myAddress)
self._send_intent(TransmitIntent(bestC, envelope.message))
return True
except InvalidActorSourceHash:
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Invalid_SourceHash)
return True
except InvalidActorSpecification:
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass)
return True
except ImportError as ex:
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Import,
errorStr=str(ex))
return True
except AttributeError as ex:
# Usually when the module has no attribute FooActor
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass,
errorStr=str(ex))
return True
except Exception as ex:
import traceback
thesplog('Exception "%s" handling PendingActor: %s',
ex,
traceback.format_exc(),
level=logging.ERROR)
self._sendPendingActorResponse(
envelope,
None,
errorCode=PendingActorResponse.ERROR_Invalid_ActorClass,
errorStr=str(ex))
return True
return super(ConventioneerAdmin, self).h_PendingActor(envelope)
def h_NotifyOnSystemRegistration(self, envelope):
if envelope.message.enableNotification:
newRegistrant = envelope.sender not in self._conventionNotificationHandlers
if newRegistrant:
self._conventionNotificationHandlers.append(envelope.sender)
# Now update the registrant on the current state of all convention members
for member in self._conventionMembers.values():
self._send_intent(
TransmitIntent(
envelope.sender,
ActorSystemConventionUpdate(
member.remoteAddress,
member.remoteCapabilities, True)))
else:
self._removeNotificationHandler(envelope.sender)
return True
def _removeNotificationHandler(self, handlerAddr):
self._conventionNotificationHandlers = [
H for H in self._conventionNotificationHandlers if H != handlerAddr
]
def h_PoisonMessage(self, envelope):
self._removeNotificationHandler(envelope.sender)
def _handleChildExited(self, childAddr):
self._removeNotificationHandler(childAddr)
return super(ConventioneerAdmin, self)._handleChildExited(childAddr)
def h_CapabilityUpdate(self, envelope):
updateLocals = self._updSystemCapabilities(
envelope.message.capabilityName, envelope.message.capabilityValue)
self.setupConvention()
if updateLocals: self._capUpdateLocalActors()
return False # might have sent with Hysteresis, so return to _run loop here
