def __init__(self, transport, address, capabilities, logdefs,
concurrency_context):
thesplog('++++ Starting Admin from %s',
sys.modules['thespian'].__file__,
level=logging.DEBUG)
super(AdminCore, self).__init__(address, transport)
self.init_replicator(transport, concurrency_context)
self.capabilities = capabilities
self.logdefs = logdefs
self._pendingChildren = {
} # Use: childLocalAddr instance # : PendingActorEnvelope
# Things that help us look like an Actor, even though we're not
self._sourceHash = None
thesplog('++++ Admin started @ %s / gen %s',
self.transport.myAddress,
str(ThespianGeneration),
level=logging.INFO,
primary=True)
logging.info('++++ Actor System gen %s started, admin @ %s',
str(ThespianGeneration), self.transport.myAddress)
logging.debug('Thespian source: %s', sys.modules['thespian'].__file__)
self._nannying = AssocList() # child actorAddress -> parent Address
self._deadLetterHandler = None
self._sources = {
} # Index is sourcehash, value PendingSource or ValidSource
self._sourceAuthority = None
self._sourceNotifications = [] # array of notification addresses
def unrecognized(self, envelope):
self._sCBStats.inc('Admin Message Received.Discarded')
thesplog("Admin got incoming %s from %s;"
" discarded because I don't know how to handle it!",
envelope.message, envelope.sender,
level=logging.WARNING, primary=True)
return True
def __init__(self, initType, *args):
super(UDPTransport, self).__init__()
if isinstance(initType, ExternalInterfaceTransportInit):
# External process that is going to talk "in". There is
# no parent, and the child is the systemAdmin.
capabilities, logDefs = args
templateAddr = UDPv4ActorAddress(None, 0)
self.socket = socket.socket(*templateAddr.socketArgs)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(*templateAddr.bindArgs)
self.myAddress = ActorAddress(UDPv4ActorAddress(*self.socket.getsockname(),
external=True))
thesplog('external template %s got actual %s', templateAddr, self.myAddress,
level=logging.DEBUG)
self._adminAddr = self.getAdminAddr(capabilities)
self._parentAddr = None
elif isinstance(initType, UDPEndpoint):
instanceNum, assignedAddr, self._parentAddr, self._adminAddr = initType.args
templateAddr = assignedAddr or ActorAddress(UDPv4ActorAddress(None, 0))
self.socket = socket.socket(*templateAddr.addressDetails.socketArgs)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(*templateAddr.addressDetails.bindArgs)
# N.B. myAddress is actually the address we will export
# for others to talk to us, not the bind address. The
# difference is that we bind to '0.0.0.0' (inaddr_any),
# but that's not a valid address for people to send stuff
# to us.
self.myAddress = ActorAddress(UDPv4ActorAddress(*self.socket.getsockname(),
external=True))
else:
thesplog('UDPTransport init of type %s unsupported', str(initType), level=logging.ERROR)
self._rcvd = []
def exec_module(self, module):
moduleName = module.__name__
hashRoot = self.finder.hashRoot()
if moduleName.startswith(hashRoot):
moduleName = moduleName[len(hashRoot):]
if self.isModuleDir:
name = ospath.join(*tuple(moduleName.split('.') + ['__init__.py']))
elif '.' in moduleName:
name = ospath.join(*tuple(moduleName.split('.'))) + '.py'
else:
name = moduleName + '.py'
codeproc = lambda s: fix_imports(s, name, hashRoot, self.finder.getZipTopLevelNames())
try:
# Ensure the file ends in a carriage-return. The path
# importer does this automatically and no trailing
# whitespace results in SyntaxError or IndentError
# exceptions. In addition, using "universal newlines"
# mode to read the file is not always effective
# (e.g. ntlm.HTTPNtlmAuthHandler.py, so explicitly ensure
# the proper line endings for the compiler.
if sys.version_info >= (3,0):
converter = lambda s: codeproc(s + b'\n')
else:
converter = lambda s: codeproc(s.replace('\r\n', '\n')+'\n')
code = self.finder.withZipElementSource(
name,
converter)
do_exec(code, module.__dict__)
except Exception as ex:
thesplog('sourceload realization failure in %s: %s',
moduleName, ex, level=logging.ERROR)
#return None
raise
def unrecognized(self, envelope):
self._sCBStats.inc('Admin Message Received.Discarded')
thesplog("Admin got incoming %s from %s;"
" discarded because I don't know how to handle it!",
envelope.message, envelope.sender,
level=logging.WARNING, primary=True)
return True
def __init__(self, transport, address, capabilities,
logdefs,
concurrency_context):
thesplog('++++ Starting Admin from %s',
sys.modules['thespian'].__file__,
level=logging.DEBUG)
super(AdminCore, self).__init__(address, transport)
self.init_replicator(transport, concurrency_context)
self.capabilities = capabilities
self.logdefs = logdefs
self._pendingChildren = {} # Use: childLocalAddr instance # : PendingActorEnvelope
# Things that help us look like an Actor, even though we're not
self._sourceHash = None
thesplog('++++ Admin started @ %s / gen %s',
self.transport.myAddress, str(ThespianGeneration),
level=logging.INFO,
primary=True)
logging.info('++++ Actor System gen %s started, admin @ %s',
str(ThespianGeneration), self.transport.myAddress)
logging.debug('Thespian source: %s', sys.modules['thespian'].__file__)
self._nannying = AssocList() # child actorAddress -> parent Address
self._deadLetterHandler = None
self._sources = {} # Index is sourcehash, value PendingSource or ValidSource
self._sourceAuthority = None
self._sourceNotifications = [] # array of notification addresses
def __init__(self, initType, *args):
super(UDPTransport, self).__init__()
templateAddr = None
if isinstance(initType, ExternalInterfaceTransportInit):
# External process that is going to talk "in". There is
# no parent, and the child is the systemAdmin.
capabilities, logDefs, concurrency_context = args
self._adminAddr = self.getAdminAddr(capabilities)
self._parentAddr = None
elif isinstance(initType, UDPEndpoint):
instanceNum, assignedAddr, self._parentAddr, self._adminAddr = initType.args
templateAddr = assignedAddr
# N.B. myAddress is actually the address we will export
# for others to talk to us, not the bind address. The
# difference is that we bind to '0.0.0.0' (inaddr_any),
# but that's not a valid address for people to send stuff
# to us.
else:
thesplog('UDPTransport init of type %s unsupported',
str(initType),
level=logging.ERROR)
if not templateAddr:
templateAddr = ActorAddress(UDPv4ActorAddress(None, 0))
self.socket = socket.socket(*templateAddr.addressDetails.socketArgs)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(*templateAddr.addressDetails.bindArgs)
self.myAddress = ActorAddress(
UDPv4ActorAddress(*self.socket.getsockname(), external=True))
self._rcvd = []
self._checkChildren = False
self._shutdownSignalled = False
self._pending_actions = [] # array of (ExpirationTimer, func)
def _schedulePreparedIntent(self, transmitIntent):
# If there's nothing to send, that's implicit success
if not transmitIntent.serMsg:
transmitIntent.result = SendStatus.Sent
transmitIntent.completionCallback()
return
# OK, this can be sent now, so go ahead and get it sent out
if not self._canSendNow():
self._aTB_queuedPendingTransmits.insert(0, transmitIntent)
if len(self._aTB_queuedPendingTransmits) >= MAX_QUEUED_TRANSMITS:
# Try to drain out local work before accepting more
# because it looks like we're getting really behind.
# This is dangerous though, because if other Actors
# are having the same issue this can create a
# deadlock.
thesplog(
'Entering tx-only mode to drain excessive queue (%s > %s, drain-to %s)',
len(self._aTB_queuedPendingTransmits),
MAX_QUEUED_TRANSMITS,
QUEUE_TRANSMIT_UNBLOCK_THRESHOLD,
level=logging.WARNING)
while len(self._aTB_queuedPendingTransmits
) > QUEUE_TRANSMIT_UNBLOCK_THRESHOLD:
self.run(TransmitOnly, transmitIntent.delay())
thesplog(
'Exited tx-only mode after draining excessive queue (%s)',
len(self._aTB_queuedPendingTransmits),
level=logging.WARNING)
return
self._submitTransmit(transmitIntent)
def _handleReadableIncoming(self, inc):
try:
rdata = inc.socket.recv(inc.remainingSize())
inc.failCount = 0
except socket.error as e:
inc.failCount = getattr(inc, 'failCount', 0) + 1
if e.errno in [errno.EAGAIN, errno.EWOULDBLOCK] and inc.failCount < MAX_CONSECUTIVE_READ_FAILURES:
inc.backoffPause(True)
return True
thesplog('Error reading from socket (#%s); closing: %s', inc.failCount, e)
return False
if not rdata:
# Since this point is only arrived at when select() says
# the socket is readable, this is an indicator of a closed
# socket. Since previous calls didn't detect
# receivedAllData(), this is an aborted/incomplete
# reception. Discard it.
return False
inc.addData(rdata)
if not inc.receivedAllData():
# Continue running and monitoring this socket
return True
try:
rEnv = ReceiveEnvelope(*inc.data)
except Exception:
import traceback
thesplog('OUCH! Error deserializing received data: %s', traceback.format_exc())
inc.socket.sendall(ackDataErrMsg)
# Continue running, but release this socket
return False
inc.socket.sendall(ackMsg)
self._incomingEnvelopes.append(rEnv)
# Continue to run, but this socket is releasable
return False
def _runWithExpiry(self, incomingHandler):
if incomingHandler == TransmitOnly or \
isinstance(incomingHandler, TransmitOnly):
# transmits are not queued/multistage in this transport, no waiting
return 0
self._aborting_run = False
while not self.run_time.expired() and not self._aborting_run:
sresp, _ign1, _ign2 = select.select([self.socket.fileno()], [], [],
self.run_time.remainingSeconds())
if [] == sresp:
if [] == _ign1 and [] == _ign2:
# Timeout, give up
return None
thesplog('Waiting for read event, but got %s %s', _ign1, _ign2, level=logging.WARNING)
continue
rawmsg, sender = self.socket.recvfrom(65535)
sendAddr = ActorAddress(UDPv4ActorAddress(*sender, external=True))
try:
msg = serializer.loads(rawmsg)
except Exception as ex:
continue
if incomingHandler is None:
return ReceiveEnvelope(sendAddr, msg)
if not incomingHandler(ReceiveEnvelope(sendAddr, msg)):
return # handler returned False, indicating run() should exit
return None
def _next_XMIT_1(self, intent):
intent.socket = socket.socket(*intent.targetAddr.addressDetails.socketArgs)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
intent.socket.setblocking(0)
# Disable Nagle to transmit headers and acks asap; our sends are usually small
intent.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
intent.socket.settimeout(timePeriodSeconds(intent.delay()))
try:
intent.socket.connect(*intent.targetAddr.addressDetails.connectArgs)
except socket.error as err:
# EINPROGRESS means non-blocking socket connect is in progress...
if err.errno != errno.EINPROGRESS:
thesplog('Socket connect failure %s to %s (returning %s)',
err, intent.targetAddr, intent.completionCallback,
level=logging.WARNING)
return self._finishIntent(intent,
SendStatus.DeadTarget \
if err.errno == errno.ECONNREFUSED \
else SendStatus.Failed)
intent.backoffPause(True)
except Exception as ex:
thesplog('Unexpected TCP socket connect exception: %s', ex,
level=logging.ERROR)
return self._finishIntent(intent, SendStatus.BadPacket)
intent.stage = self._XMITStepSendData # When connect completes
intent.amtSent = 0
return True
def _safeSocketShutdown(sock):
try:
sock.shutdown(socket.SHUT_RDWR) # KWQ: all these should be protected!
except socket.error as ex:
if ex.errno != errno.ENOTCONN:
thesplog('Error during shutdown of socket %s: %s', sock, ex)
sock.close()
def _submitTransmit(self, transmitIntent):
self._aTB_numPendingTransmits += 1
transmitIntent.addCallback(self._async_txdone, self._async_txdone)
thesplog('actualTransmit of %s', transmitIntent.identify(),
level=logging.DEBUG)
self._scheduleTransmitActual(transmitIntent)
def _checkConvention(self):
if self.isConventionLeader():
missing = []
for each in self._conventionMembers:
if self._conventionMembers[each].registryValid.expired():
missing.append(each)
for each in missing:
thesplog('%s missed %d checkins (%s); assuming it has died',
str(self._conventionMembers[each]),
CONVENTION_REGISTRATION_MISS_MAX,
str(self._conventionMembers[each].registryValid),
level=logging.WARNING, primary=True)
self._remoteSystemCleanup(self._conventionMembers[each].remoteAddress)
self._conventionRegistration = ExpiryTime(CONVENTION_REREGISTRATION_PERIOD)
else:
# Re-register with the Convention if it's time
if self._conventionAddress and self._conventionRegistration.expired():
self.setupConvention()
for each in self._conventionMembers:
member = self._conventionMembers[each]
if member.preRegistered and \
member.preRegistered.pingValid.expired() and \
not member.preRegistered.pingPending:
member.preRegistered.pingPending = True
member.preRegistered.pingValid = ExpiryTime(CONVENTION_RESTART_PERIOD
if member.registryValid.expired()
else CONVENTION_REREGISTRATION_PERIOD)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(member.remoteAddress, ConventionInvite(),
onSuccess = self._preRegQueryNotPending,
onError = self._preRegQueryNotPending))
def _missed_checkin_remote_cleanup(self, remote_member):
thesplog('%s missed %d checkins (%s); assuming it has died',
str(remote_member),
CONVENTION_REGISTRATION_MISS_MAX,
str(remote_member.registryValid),
level=logging.WARNING, primary=True)
return self._remote_system_cleanup(remote_member.remoteAddress)
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
try:
while not self.isShuttingDown():
delay = min(self._conventionRegistration or \
ExpiryTime(CONVENTION_RESTART_PERIOD
if self._conventionLost and not self.isConventionLeader() else
CONVENTION_REREGISTRATION_PERIOD),
ExpiryTime(None) if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay
)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
self.transport.run(self.handleIncoming, delay.remaining())
self._checkConvention()
self._hysteresisSender.checkSends()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(),
level=logging.ERROR, exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def _remote_system_cleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership. This is also called on
a First Time connection from the remote to discard any
previous connection information.
"""
thesplog('Convention cleanup or deregistration for %s (known? %s)',
registrant,
bool(self._conventionMembers.find(registrant)),
level=logging.INFO)
rmsgs = [LostRemote(registrant)]
cmr = self._conventionMembers.find(registrant)
if not cmr or cmr.preRegOnly:
return []
# Send exited notification to conventionNotificationHandler (if any)
for each in self._conventionNotificationHandlers:
rmsgs.append(
TransmitIntent(each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
if registrant == self.conventionLeaderAddr and self._invited:
self._conventionAddress = None
# Don't clear invited: once invited, that
# perpetually indicates this should be only a
# member and never a leader.
self._conventionMembers.rmv(registrant)
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpirationTimer(None)
cmr.preRegOnly = True
return rmsgs + [HysteresisCancel(registrant)]
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY)
txwatch = self._tx_to_admin(SystemShutdown())
for remaining_time in unexpired(time_to_quit):
response = self._run_transport(remaining_time.remaining())
if txwatch.failed:
thesplog('Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if isinstance(response, ReceiveEnvelope):
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s', response.message,
level=logging.WARNING)
elif isinstance(response, (Thespian__Run_Expired,
Thespian__Run_Terminated,
Thespian__Run_Expired)):
break
else:
thesplog('No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
self.transport.close()
thesplog('ActorSystem shutdown complete.')
def _schedulePreparedIntent(self, transmitIntent):
# If there's nothing to send, that's implicit success
if not transmitIntent.serMsg:
transmitIntent.result = SendStatus.Sent
transmitIntent.completionCallback()
return
# OK, this can be sent now, so go ahead and get it sent out
if not self._canSendNow():
self._aTB_queuedPendingTransmits.insert(0, transmitIntent)
if len(self._aTB_queuedPendingTransmits) >= MAX_QUEUED_TRANSMITS:
# Try to drain out local work before accepting more
# because it looks like we're getting really behind.
# This is dangerous though, because if other Actors
# are having the same issue this can create a
# deadlock.
thesplog('Entering tx-only mode to drain excessive queue (%s > %s, drain-to %s)',
len(self._aTB_queuedPendingTransmits),
MAX_QUEUED_TRANSMITS,
QUEUE_TRANSMIT_UNBLOCK_THRESHOLD,
level = logging.WARNING)
while len(self._aTB_queuedPendingTransmits) > QUEUE_TRANSMIT_UNBLOCK_THRESHOLD:
self.run(TransmitOnly, transmitIntent.delay())
thesplog('Exited tx-only mode after draining excessive queue (%s)',
len(self._aTB_queuedPendingTransmits),
level = logging.WARNING)
return
self._submitTransmit(transmitIntent)
def _sayGoodbye(self):
self._cleanupAdmin()
self._send_intent(TransmitIntent(self._exiting,
SystemShutdownCompleted()))
thesplog('---- shutdown completed', level=logging.INFO)
logging.info('---- Actor System shutdown')
self.shutdown_completed = True
def setup_convention(self, activation=False):
self._has_been_activated |= activation
rmsgs = []
# If not specified in capabilities, don't override any invites
# that may have been received.
self._conventionAddress = self._getConventionAddr(self.capabilities) or \
self._conventionAddress
leader_is_gone = (self._conventionMembers.find(self.conventionLeaderAddr) is None) \
if self.conventionLeaderAddr else True
if not self.isConventionLeader() and self.conventionLeaderAddr:
thesplog('Admin registering with Convention @ %s (%s)',
self.conventionLeaderAddr,
'first time' if leader_is_gone else 're-registering',
level=logging.INFO,
primary=True)
rmsgs.append(
HysteresisSend(self.conventionLeaderAddr,
ConventionRegister(self.myAddress,
self.capabilities,
leader_is_gone),
onSuccess=self._setupConventionCBGood,
onError=self._setupConventionCBError))
rmsgs.append(LogAggregator(self.conventionLeaderAddr))
self._conventionRegistration = ExpirationTimer(
CONVENTION_REREGISTRATION_PERIOD)
return rmsgs
def __init__(self, initType, *args):
super(MultiprocessQueueTransport, self).__init__()
if isinstance(initType, ExternalInterfaceTransportInit):
# External process that's going to talk "in". There is no
# parent, and the child is the systemAdmin.
capabilities, logDefs = args
self._parentQ = None
self._adminQ = Queue(MAX_ADMIN_QUEUESIZE)
self._adminAddr = self.getAdminAddr(capabilities)
self._myQAddress = ActorAddress(QueueActorAddress('~'))
self._myInputQ = Queue(MAX_ACTOR_QUEUESIZE)
elif isinstance(initType, MpQTEndpoint):
_addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr = initType.args
self._parentQ = parentQ
self._adminQ = adminQ
self._adminAddr = adminAddr
self._myQAddress = myAddr
self._myInputQ = myQueue
else:
thesplog('MultiprocessQueueTransport init of type %s unsupported!',
str(initType),
level=logging.ERROR)
# _queues is a map of direct child ActorAddresses to Queue instance. Note
# that there will be multiple keys mapping to the same Queue
# instance because routing is only either to the Parent or to
# an immediate Child.
self._queues = {}
# _fwdvia represents routing for other than immediate parent
# or child (there may be multiple target addresses mapping to
# the same forward address.
self._fwdvia = {} # key = targetAddress, value=fwdViaAddress
self._nextSubInstance = 0
def _safeSocketShutdown(sock):
try:
sock.shutdown(socket.SHUT_RDWR) # KWQ: all these should be protected!
except socket.error as ex:
if ex.errno != errno.ENOTCONN:
thesplog('Error during shutdown of socket %s: %s', sock, ex)
sock.close()
def _createInstance(self):
aClass = self._actorClass
try:
aClass = actualActorClass(
aClass,
partial(loadModuleFromHashSource, self._sourceHash,
self._sources) if self._sourceHash else None)
# Now instantiate the identified Actor class object
actorInst = withPossibleInitArgs(capabilities=self.capabilities,
requirements=self._childReqs,
globalName=self.globalName) \
.create(aClass)
self._sCBStats.inc('Actor.Instance Created')
except Exception as ex:
import traceback
logging.getLogger(str(self._actorClass)) \
.error('Actor %s @ %s instantiation exception',
self._actorClass, self.transport.myAddress,
exc_info = True)
thesplog('Actor %s @ %s instantiation exception: %s',
self._actorClass,
self.transport.myAddress,
traceback.format_exc(),
level=logging.WARNING,
primary=True)
self._sCBStats.inc('Actor.Instance Create Failed')
self._sayGoodbye()
return
self.actorInst = actorInst
self.actorInst._myRef = self
def _next_XMIT_1(self, intent):
intent.socket = socket.socket(
*intent.targetAddr.addressDetails.socketArgs)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
intent.socket.setblocking(0)
# Disable Nagle to transmit headers and acks asap; our sends are usually small
intent.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
intent.socket.settimeout(timePeriodSeconds(intent.delay()))
try:
intent.socket.connect(
*intent.targetAddr.addressDetails.connectArgs)
except socket.error as err:
# EINPROGRESS means non-blocking socket connect is in progress...
if err.errno != errno.EINPROGRESS:
thesplog('Socket connect failure %s to %s (returning %s)',
err,
intent.targetAddr,
intent.completionCallback,
level=logging.WARNING)
return self._finishIntent(intent,
SendStatus.DeadTarget \
if err.errno == errno.ECONNREFUSED \
else SendStatus.Failed)
intent.backoffPause(True)
except Exception as ex:
thesplog('Unexpected TCP socket connect exception: %s',
ex,
level=logging.ERROR)
return self._finishIntent(intent, SendStatus.BadPacket)
intent.stage = self._XMITStepSendData # When connect completes
intent.amtSent = 0
return True
def _sayGoodbye(self):
self._cleanupAdmin()
self._send_intent(
TransmitIntent(self._exiting, SystemShutdownCompleted()))
thesplog('---- shutdown completed', level=logging.INFO)
logging.info('---- Actor System shutdown')
self.shutdown_completed = True
def __init__(self, initType, *args):
super(MultiprocessQueueTransport, self).__init__()
if isinstance(initType, ExternalInterfaceTransportInit):
# External process that's going to talk "in". There is no
# parent, and the child is the systemAdmin.
capabilities, logDefs, self._concontext = args
NewQ = self._concontext.Queue if self._concontext else Queue
self._adminQ = NewQ(MAX_ADMIN_QUEUESIZE)
self._adminAddr = self.getAdminAddr(capabilities)
self._myQAddress = ActorAddress(QueueActorAddress('~'))
self._myInputQ = NewQ(MAX_ACTOR_QUEUESIZE)
self._QCore = MultiprocessQueueTCore_External(self._myInputQ, None, self._adminQ, self._adminAddr, self._myQAddress)
elif isinstance(initType, MpQTEndpoint):
_addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr, ccon = initType.args
self._concontext = ccon
self._adminQ = adminQ
self._adminAddr = adminAddr
self._myQAddress = myAddr
self._QCore = MultiprocessQueueTCore_Actor(myQueue, parentQ, adminQ, myAddr, myAddr)
elif isinstance(initType, ExternalQTransportCopy):
# External process that's going to talk "in". There is no
# parent, and the child is the systemAdmin.
self._QCore, = args
self._myQAddress = self._QCore.make_external_clone()
else:
thesplog('MultiprocessQueueTransport init of type %s unsupported!', str(initType),
level=logging.ERROR)
self._nextSubInstance = 0
def __init__(self, initType, *args):
super(MultiprocessQueueTransport, self).__init__()
if isinstance(initType, ExternalInterfaceTransportInit):
# External process that's going to talk "in". There is no
# parent, and the child is the systemAdmin.
capabilities, logDefs, self._concontext = args
NewQ = self._concontext.Queue if self._concontext else Queue
self._adminQ = NewQ(MAX_ADMIN_QUEUESIZE)
self._adminAddr = self.getAdminAddr(capabilities)
self._myQAddress = ActorAddress(QueueActorAddress('~'))
self._myInputQ = NewQ(MAX_ACTOR_QUEUESIZE)
self._QCore = MultiprocessQueueTCore_External(
self._myInputQ, None, self._adminQ, self._adminAddr,
self._myQAddress)
elif isinstance(initType, MpQTEndpoint):
_addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr, ccon = initType.args
self._concontext = ccon
self._adminQ = adminQ
self._adminAddr = adminAddr
self._myQAddress = myAddr
self._QCore = MultiprocessQueueTCore_Actor(myQueue, parentQ,
adminQ, myAddr, myAddr)
elif isinstance(initType, ExternalQTransportCopy):
# External process that's going to talk "in". There is no
# parent, and the child is the systemAdmin.
self._QCore, = args
self._myQAddress = self._QCore.make_external_clone()
else:
thesplog('MultiprocessQueueTransport init of type %s unsupported!',
str(initType),
level=logging.ERROR)
self._nextSubInstance = 0
def __init__(self, transport, address, capabilities,
logdefs,
concurrency_context):
thesplog('++++ Starting Admin from %s',
sys.modules['thespian'].__file__,
level=logging.DEBUG)
super(AdminCore, self).__init__(address, transport)
self.init_replicator(transport, concurrency_context)
self.capabilities = capabilities
self.logdefs = logdefs
self._pendingChildren = {} # Use: childLocalAddr instance # : PendingActorEnvelope
# Things that help us look like an Actor, even though we're not
self._sourceHash = None
thesplog('++++ Admin started @ %s / gen %s',
self.transport.myAddress, str(ThespianGeneration),
level=logging.INFO,
primary=True)
logging.info('++++ Actor System gen %s started, admin @ %s',
str(ThespianGeneration), self.transport.myAddress)
logging.debug('Thespian source: %s', sys.modules['thespian'].__file__)
self._nannying = AssocList() # child actorAddress -> parent Address
self._deadLetterHandler = None
self._sources = {} # Index is sourcehash, value PendingSource or ValidSource
self._sourceAuthority = None
self._sourceNotifications = [] # array of notification addresses
# The initialization of the Admin and its logger
# occurs asynchronously, but since the Admin is using a known
# address, there is nothing to prevent clients from initiating
# requests to the Admin before it has had a chance to complete
# the initialization; the _pre_init_msgs will hold those
# requests until the initialization has completed.
self._pre_init_msgs = []
def shutdown_signal_detected(signum, frame):
thesplog('Actor %s @ %s got shutdown signal: %s',
name,
addr,
signum,
level=logging.WARNING)
am.transport.interrupt_wait(signal_shutdown=True)
def setupConvention(self):
if self.isShuttingDown(): return
if not self._conventionAddress:
gCA = getattr(self.transport, 'getConventionAddress',
lambda c: None)
self._conventionAddress = gCA(self.capabilities)
if self._conventionAddress == self.myAddress:
self._conventionAddress = None
if self._conventionAddress:
thesplog('Admin registering with Convention @ %s (%s)',
self._conventionAddress,
'first time' if getattr(self, '_conventionLeaderIsGone',
True) else 're-registering',
level=logging.INFO,
primary=True)
self._hysteresisSender.sendWithHysteresis(
TransmitIntent(self._conventionAddress,
ConventionRegister(
self.myAddress, self.capabilities,
getattr(self, '_conventionLeaderIsGone',
True)),
onSuccess=self._setupConventionCBGood,
onError=self._setupConventionCBError))
self._conventionRegistration = ExpiryTime(
CONVENTION_REREGISTRATION_PERIOD)
def _remoteSystemCleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership.
"""
thesplog('Convention cleanup or deregistration for %s (new? %s)',
registrant,
registrant.actorAddressString not in self._conventionMembers,
level=logging.INFO)
if registrant.actorAddressString in self._conventionMembers:
cmr = self._conventionMembers[registrant.actorAddressString]
# Send exited notification to conventionNotificationHandler (if any)
if self.isConventionLeader():
for each in self._conventionNotificationHandlers:
self._send_intent(
TransmitIntent(
each,
ActorSystemConventionUpdate(
cmr.remoteAddress, cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
self._send_intent(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
del self._conventionMembers[registrant.actorAddressString]
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpiryTime(None)
if registrant == self._conventionAddress:
# Convention Leader has exited. Do NOT set
# conventionAddress to None. It might speed up shutdown
# of this ActorSystem because it won't try to de-register
# from the convention leader, but if the convention leader
# reappears there will be nothing to get this ActorSystem
# re-registered with the convention.
self._conventionLeaderIsGone = True
self._hysteresisSender.cancelSends(registrant)
def prepMessageSend(self, anAddress, msg):
"""Prepares to send the specified message to the specified address,
returning a tuple of the send-to-address and the (possibly
updated) message to send.
The address may be converted from an internal to an
exportable address.
If the target address is known as a dead letter box, the
Admin address is returned instead and the message is
wrapped in a DeadEnvelope wrapper.
If the target address is not ready for use, the
send-to-address portion of the tuple will return a value of
None.
If the message should no longer be sent, the message
portion of the tuple will be returned as
SendStatus.DeadTarget (because this is *never* a valid
message to actually send).
n.b. this method may or may not be called while holding a
lock. It is a lookup-only operation, so the lock should
not be of any consequence.
"""
tgtaddr = self.exportAddr(anAddress)
if tgtaddr is None:
return None, msg
if tgtaddr in self._deadAddrs:
if isinstance(msg, (DeadEnvelope, ChildActorExited)):
thesplog('Discarding %s to %s because the latter is dead.',
str(msg), str(tgtaddr))
return None, SendStatus.DeadTarget
return self._adminAddr, DeadEnvelope(anAddress, msg)
return tgtaddr, msg
def prepMessageSend(self, anAddress, msg):
"""Prepares to send the specified message to the specified address,
returning a tuple of the send-to-address and the (possibly
updated) message to send.
The address may be converted from an internal to an
exportable address.
If the target address is known as a dead letter box, the
Admin address is returned instead and the message is
wrapped in a DeadEnvelope wrapper.
If the target address is not ready for use, the
send-to-address portion of the tuple will return a value of
None.
If the message should no longer be sent, the message
portion of the tuple will be returned as
SendStatus.DeadTarget (because this is *never* a valid
message to actually send).
"""
tgtaddr = self.exportAddr(anAddress)
if tgtaddr is None:
return None, msg
if tgtaddr in self._deadAddrs:
if isinstance(msg, (DeadEnvelope, ChildActorExited)):
thesplog('Discarding %s to %s because the latter is dead.',
str(msg), str(tgtaddr))
return None, SendStatus.DeadTarget
return self._adminAddr, DeadEnvelope(anAddress, msg)
return tgtaddr, msg
def _pendingActorResponse(self, envelope):
# Have seen it arrive here without errorCode set on the PendingActorResponse...
if not hasattr(envelope.message, 'errorCode'):
thesplog('Corrupted Pending Actor Response?: %s (%s)',
envelope.message,
dir(envelope.message),
level=logging.ERROR)
return True
if not getattr(envelope.message, 'errorCode', 'Failed'):
self._sCBStats.inc('Actor.Child.Requested.Success')
self._pendingActorReady(envelope.message.instanceNum,
envelope.message.actualAddress,
isMyChild=not envelope.message.globalName)
return True
# Pending Actor Creation failed, clean up all the stuff associated with the intended Actor
self._sCBStats.inc('Actor.Child.Requested.Failure')
thesplog('Pending Actor create for %s failed (%s): %s',
envelope.message.forActor,
getattr(envelope.message, 'errorCode', '??'),
getattr(envelope.message, 'errorStr', '---'))
logging.getLogger(str(self._actorClass)) \
.error('Pending Actor create for %s failed (%s): %s',
envelope.message.forActor,
getattr(envelope.message, 'errorCode', '??'),
getattr(envelope.message, 'errorStr', '---'))
# Cancel any queued transmits for this child.
self._retryPendingChildOperations(envelope.message.instanceNum, None)
return True
def change_address_for_transmit(self, oldaddr, newaddr):
oldidx = self._atd.find(oldaddr)
if oldidx is None:
# Have not scheduled any transmits for this (probably new)
# child yet.
return
newidx = self._atd.find(newaddr)
if newidx is None:
self._atd.add(newaddr, oldidx)
elif newidx != oldidx:
if isinstance(oldaddr.addressDetails, ActorLocalAddress):
# This can happen if sends are made to createActor
# results with a globalName before the actual address
# is known. Each createActor creates a local address,
# but all those local addresses map back to the same
# actual address.
self._ptl[newidx].extend(self._ptl[oldidx])
self._atd.add(oldaddr, newidx)
self._ptl[oldidx] = [] # should not be used anymore
else:
thesplog(
'Duplicate pending transmit indices'
': %s -> %s, %s -> %s',
oldaddr,
oldidx,
newaddr,
newidx,
level=logging.ERROR)
def __init__(self, childClass, globalName, transport, sourceHash,
sourceToLoad, parentAddr, adminAddr, childRequirements,
currentSystemCapabilities, concurrency_context):
super(ActorManager, self).__init__(adminAddr, transport)
self.init_replicator(transport, concurrency_context)
self._parentAddr = parentAddr
self._sourceHash = sourceHash
self._sources = {sourceHash: sourceToLoad}
# Cache the current system capabilities to use for createActor
# attempts.
self.capabilities = currentSystemCapabilities
self._actorClass = childClass # nb. this may be a string, and sourceHash is not loaded yet
self._childReqs = childRequirements
self.actorInst = None
self.globalName = globalName
self._srcNotifyEnabled = False
atexit.register(self._shutdownActor)
thesplog('Starting Actor %s at %s (parent %s, admin %s, srcHash %s)',
childClass,
self.transport.myAddress,
self._parentAddr,
adminAddr,
self._sourceHash,
level=logging.INFO,
primary=True)
def __init__(self, initType, *args):
super(MultiprocessQueueTransport, self).__init__()
if isinstance(initType, ExternalInterfaceTransportInit):
# External process that's going to talk "in". There is no
# parent, and the child is the systemAdmin.
capabilities, logDefs = args
self._parentQ = None
self._adminQ = Queue(MAX_ADMIN_QUEUESIZE)
self._adminAddr = self.getAdminAddr(capabilities)
self._myQAddress = ActorAddress(QueueActorAddress('~'))
self._myInputQ = Queue(MAX_ACTOR_QUEUESIZE)
elif isinstance(initType, MpQTEndpoint):
_addrInst, myAddr, myQueue, parentQ, adminQ, adminAddr = initType.args
self._parentQ = parentQ
self._adminQ = adminQ
self._adminAddr = adminAddr
self._myQAddress = myAddr
self._myInputQ = myQueue
else:
thesplog('MultiprocessQueueTransport init of type %s unsupported!', str(initType),
level=logging.ERROR)
# _queues is a map of direct child ActorAddresses to Queue instance. Note
# that there will be multiple keys mapping to the same Queue
# instance because routing is only either to the Parent or to
# an immediate Child.
self._queues = {}
# _fwdvia represents routing for other than immediate parent
# or child (there may be multiple target addresses mapping to
# the same forward address.
self._fwdvia = {} # key = targetAddress, value=fwdViaAddress
self._nextSubInstance = 0
def _pendingActorReady(self, childInstance, actualAddress):
if childInstance not in self._pendingChildren:
thesplog(
'Pending actor is ready at %s for UNKNOWN %s'
'; sending child a shutdown',
actualAddress,
childInstance,
level=logging.WARNING)
self._send_intent(
TransmitIntent(actualAddress,
ActorExitRequest(recursive=True)))
return
requestEnvelope = self._pendingChildren[childInstance]
del self._pendingChildren[childInstance]
if requestEnvelope.message.globalName or \
not requestEnvelope.message.forActor:
# The Admin is the responsible Parent for these children
self._registerChild(actualAddress)
else:
# Anything the Admin was requested to create is a adoptive
# child and should be killed when the Admin exits.
self._registerChild(actualAddress)
if requestEnvelope.message.forActor:
# Proxy-parenting; remember the real parent
self._nannying.add(actualAddress, requestEnvelope.message.forActor)
self._addrManager.associateUseableAddress(self.myAddress,
childInstance, actualAddress)
# n.b. childInstance is for this Admin, but caller's
# childInstance is in original request
self._sendPendingActorResponse(requestEnvelope, actualAddress)
self._retryPendingChildOperations(childInstance, actualAddress)
def _scheduleTransmitActual(self, transmitIntent):
try:
if transmitIntent.targetAddr == self.myAddress:
if transmitIntent.message:
self._myInputQ.put( (self._myQAddress, transmitIntent.serMsg), True,
timePeriodSeconds(transmitIntent.delay()))
else:
tgtQ = self._queues.find(transmitIntent.targetAddr)
if tgtQ:
tgtQ.put((self._myQAddress, transmitIntent.serMsg), True,
timePeriodSeconds(transmitIntent.delay()))
else:
# None means sent by parent, so don't send BACK to parent if unknown
topOrFromBelow = self._myQAddress if self._parentQ else None
(self._parentQ or self._adminQ).put(
(topOrFromBelow, transmitIntent.serMsg),
True,
timePeriodSeconds(transmitIntent.delay()))
transmitIntent.tx_done(SendStatus.Sent)
return
except Q.Full:
pass
transmitIntent.tx_done(SendStatus.DeadTarget if not isinstance(
transmitIntent._message,
(ChildActorExited, ActorExitRequest)) else SendStatus.Failed)
thesplog('Q.Full %s to %s result %s', transmitIntent._message, transmitIntent.targetAddr, transmitIntent.result)
def __init__(self, initType, *args):
super(UDPTransport, self).__init__()
templateAddr = None
if isinstance(initType, ExternalInterfaceTransportInit):
# External process that is going to talk "in". There is
# no parent, and the child is the systemAdmin.
capabilities, logDefs, concurrency_context = args
self._adminAddr = self.getAdminAddr(capabilities)
self._parentAddr = None
elif isinstance(initType, UDPEndpoint):
instanceNum, assignedAddr, self._parentAddr, self._adminAddr = initType.args
templateAddr = assignedAddr
# N.B. myAddress is actually the address we will export
# for others to talk to us, not the bind address. The
# difference is that we bind to '0.0.0.0' (inaddr_any),
# but that's not a valid address for people to send stuff
# to us.
else:
thesplog('UDPTransport init of type %s unsupported', str(initType), level=logging.ERROR)
if not templateAddr:
templateAddr = ActorAddress(UDPv4ActorAddress(None, 0))
self.socket = socket.socket(*templateAddr.addressDetails.socketArgs)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(*templateAddr.addressDetails.bindArgs)
self.myAddress = ActorAddress(UDPv4ActorAddress(*self.socket.getsockname(),
external=True))
self._rcvd = []
self._checkChildren = False
self._shutdownSignalled = False
self._pending_actions = [] # array of (ExpirationTimer, func)
def _createInstance(self):
aClass = self._actorClass
try:
aClass = actualActorClass(
aClass,
partial(loadModuleFromHashSource, self._sourceHash,
self._sources) if self._sourceHash else None)
# Now instantiate the identified Actor class object
actorInst = aClass()
self._sCBStats.inc('Actor.Instance Created')
except Exception as ex:
import traceback
logging.getLogger(str(self._actorClass)) \
.error('Actor %s @ %s instantiation exception',
self._actorClass, self.transport.myAddress,
exc_info = True)
thesplog('Actor %s @ %s instantiation exception: %s',
self._actorClass,
self.transport.myAddress,
traceback.format_exc(),
level=logging.ERROR,
primary=True)
self._sCBStats.inc('Actor.Instance Create Failed')
self._sayGoodbye()
return
self.actorInst = actorInst
self.actorInst._myRef = self
def _remote_system_cleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership. This is also called on
a First Time connection from the remote to discard any
previous connection information.
"""
thesplog('Convention cleanup or deregistration for %s (known? %s)',
registrant,
bool(self._conventionMembers.find(registrant)),
level=logging.INFO)
rmsgs = [LostRemote(registrant)]
cmr = self._conventionMembers.find(registrant)
if not cmr or cmr.preRegOnly:
return []
# Send exited notification to conventionNotificationHandler (if any)
for each in self._conventionNotificationHandlers:
rmsgs.append(
TransmitIntent(each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
rmsgs.append(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
if registrant == self.conventionLeaderAddr and self._invited:
self._conventionAddress = None
# Don't clear invited: once invited, that
# perpetually indicates this should be only a
# member and never a leader.
self._conventionMembers.rmv(registrant)
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpirationTimer(None)
cmr.preRegOnly = True
return rmsgs + [HysteresisCancel(registrant)]
def _pendingActorReady(self, childInstance, actualAddress):
if childInstance not in self._pendingChildren:
thesplog('Pending actor is ready at %s for %s but latter is unknown'
'; sending child a shutdown',
actualAddress, childInstance, level=logging.WARNING)
self._send_intent(
TransmitIntent(actualAddress, ActorExitRequest(recursive=True)))
return
requestEnvelope = self._pendingChildren[childInstance]
del self._pendingChildren[childInstance]
if requestEnvelope.message.globalName or \
not requestEnvelope.message.forActor:
# The Admin is the responsible Parent for these children
self._registerChild(actualAddress)
else:
# Anything the Admin was requested to create is a adoptive
# child and should be killed when the Admin exits.
self._registerChild(actualAddress)
if requestEnvelope.message.forActor:
# Proxy-parenting; remember the real parent
self._nannying[actualAddress] = requestEnvelope.message.forActor
self._addrManager.associateUseableAddress(self.myAddress, childInstance, actualAddress)
# n.b. childInstance is for this Admin, but caller's childInstance is in original request
self._sendPendingActorResponse(requestEnvelope, actualAddress)
self._retryPendingChildOperations(childInstance, actualAddress)
def _createInstance(self):
aClass = self._actorClass
try:
aClass = actualActorClass(aClass,
partial(loadModuleFromHashSource,
self._sourceHash,
self._sources)
if self._sourceHash else None)
# Now instantiate the identified Actor class object
actorInst = aClass()
self._sCBStats.inc('Actor.Instance Created')
except Exception as ex:
import traceback
logging.getLogger(str(self._actorClass)) \
.error('Actor %s @ %s instantiation exception',
self._actorClass, self.transport.myAddress,
exc_info = True)
thesplog('Actor %s @ %s instantiation exception: %s', self._actorClass,
self.transport.myAddress, traceback.format_exc(),
level=logging.ERROR, primary=True)
self._sCBStats.inc('Actor.Instance Create Failed')
self._sayGoodbye()
return
self.actorInst = actorInst
self.actorInst._myRef = self
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
transport_continue = True
try:
while not getattr(self, 'shutdown_completed', False) and \
not isinstance(transport_continue, Thespian__Run_Terminated):
ct = currentTime()
delay = min(
self._cstate.convention_inattention_delay(ct),
ExpirationTimer(None).view(ct)
if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
transport_continue = self.transport.run(
self.handleIncoming, delay.remaining())
# Check Convention status based on the elapsed time
self._performIO(self._cstate.check_convention())
self._hysteresisSender.checkSends()
self._remove_expired_sources()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s',
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def h_SystemShutdown(self, envelope):
self._exiting = envelope.sender
thesplog('---- shutdown initiated by %s', envelope.sender, level=logging.DEBUG)
# Send failure notices and clear out any pending children. If
# any pending child ready notifications are received after
# this, they will automatically be sent an ActorExitRequest.
for each in self._pendingChildren:
pendingReq = self._pendingChildren[each]
self._send_intent(
TransmitIntent(
pendingReq.sender,
PendingActorResponse(
pendingReq.message.forActor,
pendingReq.message.instanceNum,
pendingReq.message.globalName,
errorCode = PendingActorResponse.ERROR_ActorSystem_Shutting_Down)))
self._pendingChildren = []
if not self.childAddresses: # no children?
self._sayGoodbye()
self.transport.abort_run(drain=True)
return True
# Now shutdown any direct children
self._killLocalActors()
# Once children confirm their exits the callback will shutdown the Admin.
return True
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpiryTime(MAX_SYSTEM_SHUTDOWN_DELAY)
self.transport.scheduleTransmit(
None,
TransmitIntent(self.adminAddr,
SystemShutdown(),
onError=self._shutdownSendFailed))
while not time_to_quit.expired():
response = self.transport.run(None, time_to_quit.remaining())
if getattr(self, '_TASF', False):
thesplog(
'Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if response:
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s',
response.message,
level=logging.WARNING)
else:
thesplog(
'No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
thesplog('ActorSystem shutdown complete.')
def exec_module(self, module):
moduleName = module.__name__
hashRoot = self.finder.hashRoot() + '.'
if moduleName.startswith(hashRoot):
moduleName = moduleName[len(hashRoot):]
if self.isModuleDir:
name = ospath.join(*tuple(moduleName.split('.') + ['__init__.py']))
elif '.' in moduleName:
name = ospath.join(*tuple(moduleName.split('.'))) + '.py'
else:
name = moduleName + '.py'
codeproc = lambda s: fix_imports(s, name, hashRoot,
self.finder.getZipTopLevelNames())
try:
# Ensure the file ends in a carriage-return. The path
# importer does this automatically and no trailing
# whitespace results in SyntaxError or IndentError
# exceptions. In addition, using "universal newlines"
# mode to read the file is not always effective
# (e.g. ntlm.HTTPNtlmAuthHandler.py, so explicitly ensure
# the proper line endings for the compiler.
if sys.version_info >= (3, 0):
converter = lambda s: codeproc(s + b'\n')
else:
converter = lambda s: codeproc(s.replace('\r\n', '\n') + '\n')
code = self.finder.withZipElementSource(name, converter)
do_exec(code, module.__dict__)
except Exception as ex:
thesplog('sourceload realization failure: %s',
ex,
level=logging.ERROR)
#return None
raise
def shutdown(self):
thesplog('ActorSystem shutdown requested.', level=logging.INFO)
time_to_quit = ExpirationTimer(MAX_SYSTEM_SHUTDOWN_DELAY)
txwatch = self._tx_to_admin(SystemShutdown())
while not time_to_quit.expired():
response = self._run_transport(time_to_quit.remaining())
if txwatch.failed:
thesplog('Could not send shutdown request to Admin'
'; aborting but not necessarily stopped',
level=logging.WARNING)
return
if isinstance(response, ReceiveEnvelope):
if isinstance(response.message, SystemShutdownCompleted):
break
else:
thesplog('Expected shutdown completed message, got: %s', response.message,
level=logging.WARNING)
elif isinstance(response, (Thespian__Run_Expired,
Thespian__Run_Terminated,
Thespian__Run_Expired)):
break
else:
thesplog('No response to Admin shutdown request; Actor system not completely shutdown',
level=logging.ERROR)
self.transport.close()
thesplog('ActorSystem shutdown complete.')
def _checkNextTransmit(self, result, completedIntent):
# This is the callback for (all) TransmitIntents that will
# send the next queued intent for that destination.
if completedIntent.nextIntent:
self._send_intent_to_transport(completedIntent.nextIntent)
else:
fkey = completedIntent.targetAddr
if fkey not in self._finalTransmitPending:
fkey = self._addrManager.sendToAddress(completedIntent.targetAddr)
if fkey not in self._finalTransmitPending:
if isinstance(completedIntent.message, DeadEnvelope):
fkey = completedIntent.message.deadAddress
if fkey not in self._finalTransmitPending:
fkey = self._addrManager.sendToAddress(fkey)
if fkey in self._finalTransmitPending:
if self._finalTransmitPending[fkey] != completedIntent:
thesplog('Completed final intent %s does not match recorded final intent: %s',
completedIntent.identify(),
self._finalTransmitPending[fkey].identify(),
level=logging.WARNING)
del self._finalTransmitPending[fkey]
else:
thesplog('Completed Transmit Intent %s for unrecorded destination %s / %s in %s',
completedIntent.identify(),
str(self._addrManager.sendToAddress(completedIntent.targetAddr)),
fkey,
str(map(str,self._finalTransmitPending.keys())),
level=logging.WARNING)
self._sCBStats.inc('Action.Message Send.Unknown Completion')
return
def run(self):
# Main loop for convention management. Wraps the lower-level
# transport with a stop at the next needed convention
# registration period to re-register.
transport_continue = True
try:
while not getattr(self, 'shutdown_completed', False) and \
not isinstance(transport_continue, Thespian__Run_Terminated):
delay = min(self._cstate.convention_inattention_delay(),
ExpirationTimer(None) if self._hysteresisSender.delay.expired() else
self._hysteresisSender.delay
)
# n.b. delay does not account for soon-to-expire
# pingValids, but since delay will not be longer than
# a CONVENTION_REREGISTRATION_PERIOD, the worst case
# is a doubling of a pingValid period (which should be fine).
transport_continue = self.transport.run(self.handleIncoming,
delay.remaining())
# Check Convention status based on the elapsed time
self._performIO(self._cstate.check_convention())
self._hysteresisSender.checkSends()
self._remove_expired_sources()
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(),
level=logging.ERROR, exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def h_SystemShutdown(self, envelope):
self._exiting = envelope.sender
thesplog('---- shutdown initiated by %s',
envelope.sender,
level=logging.DEBUG)
# Send failure notices and clear out any pending children. If
# any pending child ready notifications are received after
# this, they will automatically be sent an ActorExitRequest.
for each in self._pendingChildren:
pendingReq = self._pendingChildren[each]
self._send_intent(
TransmitIntent(
pendingReq.sender,
PendingActorResponse(pendingReq.message.forActor,
pendingReq.message.instanceNum,
pendingReq.message.globalName,
errorCode=PendingActorResponse.
ERROR_ActorSystem_Shutting_Down)))
self._pendingChildren = []
if not self.childAddresses: # no children?
self._sayGoodbye()
self.transport.abort_run(drain=True)
return True
# Now shutdown any direct children
self._killLocalActors()
# Callback will shutdown the Admin Once the children confirm
# their exits.
return True
def _missed_checkin_remote_cleanup(self, remote_member):
thesplog('%s missed %d checkins (%s); assuming it has died',
str(remote_member),
CONVENTION_REGISTRATION_MISS_MAX,
str(remote_member.registryValid),
level=logging.WARNING, primary=True)
return self._remote_system_cleanup(remote_member.remoteAddress)
def run(self):
if self.actorInst is None: self._createInstance()
if self.actorInst:
try:
while True:
r = self.transport.run(self.handleMessages)
if isinstance(r, Thespian__UpdateWork):
self._send_intent(TransmitIntent(
self.myAddress, r)) # tickle the transmit queues
continue
# Expects that on completion of self.transport.run
# that the Actor is done processing and that it has
# been shutdown gracefully.
break
except Exception as ex:
# This is usually an internal problem, since the
# request handling itself catches any exceptions from
# the Actor itself.
import traceback
thesplog('Actor %s @ %s transport run exception: %s',
self._actorClass,
self.transport.myAddress,
traceback.format_exc(),
level=logging.ERROR,
exc_info=True)
self._shutdownActor(True)
self.drainTransmits()
else:
self.drainTransmits()
thesplog('Run %s done', self._actorClass, level=logging.DEBUG)
def _runWithExpiry(self, incomingHandler):
if incomingHandler == TransmitOnly or \
isinstance(incomingHandler, TransmitOnly):
# transmits are not queued/multistage in this transport, no waiting
return 0
self._aborting_run = False
while not self.run_time.expired() and not self._aborting_run:
if self._rcvd:
rcvdEnv = self._rcvd.pop()
else:
next_action_timeout = self.check_pending_actions()
try:
sresp, _ign1, _ign2 = select.select([self.socket.fileno()], [], [],
min(self.run_time, next_action_timeout)
.remainingSeconds())
except select.error as se:
import errno
if se.args[0] != errno.EINTR:
thesplog('Error during select: %s', se)
return None
continue
except ValueError:
# self.run_time can expire between the while test
# and the use in the select statement.
continue
if [] == sresp:
if [] == _ign1 and [] == _ign2:
# Timeout, give up
return None
thesplog('Waiting for read event, but got %s %s', _ign1, _ign2, level=logging.WARNING)
continue
rawmsg, sender = self.socket.recvfrom(65535)
if rawmsg == b'BuMP':
sendAddr = self.myAddress
if self._checkChildren:
self._checkChildren = False
msg = ChildMayHaveDied()
elif self._shutdownSignalled:
self._shutdownSignalled = False
msg = ActorExitRequest()
else:
return Thespian__UpdateWork()
else:
sendAddr = ActorAddress(UDPv4ActorAddress(*sender, external=True))
try:
msg = serializer.loads(rawmsg)
except Exception:
continue
rcvdEnv = ReceiveEnvelope(sendAddr, msg)
if incomingHandler is None:
return rcvdEnv
r = incomingHandler(rcvdEnv)
if not r:
return r # handler returned False, indicating run() should exit
return None
def _remoteSystemCleanup(self, registrant):
"""Called when a RemoteActorSystem has exited and all associated
Actors should be marked as exited and the ActorSystem
removed from Convention membership.
"""
thesplog('Convention cleanup or deregistration for %s (new? %s)',
registrant,
registrant.actorAddressString not in self._conventionMembers,
level=logging.INFO)
if registrant.actorAddressString in self._conventionMembers:
cmr = self._conventionMembers[registrant.actorAddressString]
# Send exited notification to conventionNotificationHandler (if any)
if self.isConventionLeader():
for each in self._conventionNotificationHandlers:
self._send_intent(
TransmitIntent(each,
ActorSystemConventionUpdate(cmr.remoteAddress,
cmr.remoteCapabilities,
False))) # errors ignored
# If the remote ActorSystem shutdown gracefully (i.e. sent
# a Convention Deregistration) then it should not be
# necessary to shutdown remote Actors (or notify of their
# shutdown) because the remote ActorSystem should already
# have caused this to occur. However, it won't hurt, and
# it's necessary if the remote ActorSystem did not exit
# gracefully.
for lpa, raa in cmr.hasRemoteActors:
# ignore errors:
self._send_intent(TransmitIntent(lpa, ChildActorExited(raa)))
# n.b. at present, this means that the parent might
# get duplicate notifications of ChildActorExited; it
# is expected that Actors can handle this.
# Remove remote system from conventionMembers
if not cmr.preRegistered:
del self._conventionMembers[registrant.actorAddressString]
else:
# This conventionMember needs to stay because the
# current system needs to continue issuing
# registration pings. By setting the registryValid
# expiration to forever, this member won't re-time-out
# and will therefore be otherwise ignored... until it
# registers again at which point the membership will
# be updated with new settings.
cmr.registryValid = ExpiryTime(None)
if registrant == self._conventionAddress:
# Convention Leader has exited. Do NOT set
# conventionAddress to None. It might speed up shutdown
# of this ActorSystem because it won't try to de-register
# from the convention leader, but if the convention leader
# reappears there will be nothing to get this ActorSystem
# re-registered with the convention.
self._conventionLeaderIsGone = True
self._hysteresisSender.cancelSends(registrant)
def run(self):
try:
self.transport.run(self.handleIncoming, None)
except Exception as ex:
import traceback
thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(),
level=logging.ERROR, exc_info=True)
thesplog('Admin time to die', level=logging.DEBUG)
def scheduleTransmit(self, addressManager, transmitIntent):
"""Requests that a transmit be performed. The message and target
address must be fully valid at this point; any local
addresses should throw a CannotPickleAddress exception and
the caller is responsible for retrying later when those
addresses are available.
If addressManager is None then the intent address is
assumed to be valid but it cannot be updated if it is a
local address or a dead address. A value of None is
normally only used at Admin or Actor startup time when
confirming the established connection back to the parent,
at which time the target address should always be valid.
Any transmit attempts from a thread other than the main
thread are queued; calls to the underlying transmit layer
are done only from the context of the main thread.
"""
if addressManager:
# Verify the target address is useable
targetAddr, txmsg = addressManager.prepMessageSend(
transmitIntent.targetAddr,
transmitIntent.message)
try:
isDead = txmsg == SendStatus.DeadTarget
except Exception:
# txmsg may have an __eq__ that caused an exception
isDead = False
if isDead:
# Address Manager has indicated that these messages
# should never be attempted because the target is
# dead. This is *only* for special messages like
# DeadEnvelope and ChildActorExited which would
# endlessly recurse or bounce back and forth. This
# code indicates here that the transmit was
# "successful" to allow normal cleanup but to avoid
# recursive error generation.
thesplog('Faking dead target transmit result Sent for %s',
transmitIntent, level=logging.WARNING)
transmitIntent.tx_done(SendStatus.Sent)
return
if not targetAddr:
raise CannotPickleAddress(transmitIntent.targetAddr)
# In case the prep made some changes...
transmitIntent.changeTargetAddr(targetAddr)
transmitIntent.changeMessage(txmsg)
# Verify that the message can be serialized. This may throw
# an exception, which will cause the caller to store this
# intent and retry it at some future point (the code up to and
# including this serialization should be idempotent).
transmitIntent.serMsg = self.serializer(transmitIntent)
self._schedulePreparedIntent(transmitIntent)
def h_EndpointConnected(self, envelope):
for C in getattr(self, '_child_procs', []):
if envelope.message.childInstance == C.childNum:
C.childRealAddr = envelope.sender
break
else:
thesplog('Unknown child process endpoint connected: %s', envelope, level=logging.WARNING)
self._pendingActorReady(envelope.message.childInstance, envelope.sender)
return True