def startupASLogger(addrOfStarter, logEndpoint, logDefs,
transportClass, aggregatorAddress):
# Dirty trick here to completely re-initialize logging in this
# process... something the standard Python logging interface does
# not allow via the API. We also do not want to run
# logging.shutdown() because (a) that does not do enough to reset,
# and (b) it shuts down handlers, but we want to leave the
# parent's handlers alone. Dirty trick here to completely
# re-initialize logging in this process... something the standard
# Python logging interface does not allow via the API.
logging.root = logging.RootLogger(logging.WARNING)
logging.Logger.root = logging.root
logging.Logger.manager = logging.Manager(logging.Logger.root)
if logDefs:
dictConfig(logDefs)
else:
logging.basicConfig()
# Disable thesplog from within the logging process (by setting the
# logfile size to zero) to try to avoid recursive logging loops.
thesplog_control(logging.WARNING, False, 0)
#logging.info('ActorSystem Logging Initialized')
transport = transportClass(logEndpoint)
setProcName('logger', transport.myAddress)
transport.scheduleTransmit(None, TransmitIntent(addrOfStarter, LoggerConnected()))
fdup = None
last_exception = None
last_exception_time = None
exception_count = 0
while True:
try:
r = transport.run(None)
logrecord = r.message
if isinstance(logrecord, LoggerExitRequest):
logging.info('ActorSystem Logging Shutdown')
return
elif isinstance(logrecord, LoggerFileDup):
fdup = getattr(logrecord, 'fname', None)
elif isinstance(logrecord, LogAggregator):
aggregatorAddress = logrecord.aggregatorAddress
elif isinstance(logrecord, logging.LogRecord):
logging.getLogger(logrecord.name).handle(logrecord)
if fdup:
with open(fdup, 'a') as ldf: ldf.write('%s\n'%str(logrecord))
if aggregatorAddress and \
logrecord.levelno >= logging.WARNING:
transport.scheduleTransmit(None, TransmitIntent(aggregatorAddress,
logrecord))
else:
logging.warn('Unknown message rcvd by logger: %s'%str(logrecord))
except Exception:
logging.error('Thespian Logger aborting (#%d) with error', exception_count, exc_info=True)
if last_exception is None or datetime.now() - last_exception_time > timedelta(seconds=1):
last_exception_time = datetime.now()
exception_count = 0
else:
exception_count += 1
if exception_count >= MAX_LOGGING_EXCEPTIONS_PER_SECOND:
logging.error('Too many Thespian Logger exceptions (#%d in %s); exiting!',
exception_count, datetime.now() - last_exception_time)
return
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass,
adminAddr, capabilities, logDefs, concurrency_context):
# Unix Daemonization; skipped if not available
import os, sys
if hasattr(os, 'setsid'):
os.setsid()
try:
import resource
resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # No core dumps
except Exception:
pass
if hasattr(os, 'fork'):
if os.fork(): sys.exit(0)
# Slight trickiness here. There may *already* be an admin bound
# to this start address. However, the external process attempting
# to start is going to wait for the EndpointConnected message
# before continuing, so ensure that message is *not* sent until
# the local admin Transport has had time to bind and listen to the
# local address, but also ensure that the message is *always* sent
# even if the local admin could not start (caller will use
# _verifyAdminRunning to ensure things are OK.
transport = transportClass(endpointPrep)
try:
admin = adminClass(transport, adminAddr, capabilities, logDefs,
concurrency_context)
except Exception:
transport.scheduleTransmit(
None, TransmitIntent(addrOfStarter, EndpointConnected(0)))
raise
# Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below.
admin.addrOfStarter = addrOfStarter
setProcName(
adminClass.__name__.rpartition('.')[-1], admin.transport.myAddress)
# Admin does not do normal signal handling, but does want to know if children exit
for each in range(1, signal.NSIG):
# n.b. normally Python intercepts SIGINT to turn it into a
# KeyboardInterrupt exception. However, these Actors should
# be detached from the keyboard, so revert to normal SIGINT
# behavior.
if each not in uncatchable_signals:
if each in child_exit_signals:
set_signal_handler(each, admin.signalChildDied)
if hasattr(signal, 'SIGUSR1'):
set_signal_handler(signal.SIGUSR1, signal_admin_sts(admin))
_startLogger(transportClass, transport, admin, capabilities, logDefs,
concurrency_context)
#closeUnusedFiles(transport)
# Admin should never enter TX-only flow control state because this
# could deadlock or other non-progress conditions, especially if
# using admin routing.
transport.enableRXPauseFlowControl(False)
admin.run()
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass,
adminAddr, capabilities, logDefs, concurrency_context):
# Unix Daemonization; skipped if not available
import os,sys
if hasattr(os, 'setsid'):
os.setsid()
try:
import resource
resource.setrlimit(resource.RLIMIT_CORE, (0,0)) # No core dumps
except Exception: pass
if hasattr(os, 'fork'):
if os.fork(): sys.exit(0)
# Slight trickiness here. There may *already* be an admin bound
# to this start address. However, the external process attempting
# to start is going to wait for the EndpointConnected message
# before continuing, so ensure that message is *not* sent until
# the local admin Transport has had time to bind and listen to the
# local address, but also ensure that the message is *always* sent
# even if the local admin could not start (caller will use
# _verifyAdminRunning to ensure things are OK.
transport = transportClass(endpointPrep)
try:
admin = adminClass(transport, adminAddr, capabilities, logDefs,
concurrency_context)
except Exception:
transport.scheduleTransmit(None,
TransmitIntent(addrOfStarter, EndpointConnected(0)))
raise
# Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below.
admin.addrOfStarter = addrOfStarter
setProcName(adminClass.__name__.rpartition('.')[-1],
admin.transport.myAddress)
# Admin does not do normal signal handling, but does want to know if children exit
for each in range(1, signal.NSIG):
# n.b. normally Python intercepts SIGINT to turn it into a
# KeyboardInterrupt exception. However, these Actors should
# be detached from the keyboard, so revert to normal SIGINT
# behavior.
if each not in uncatchable_signals:
if each in child_exit_signals:
set_signal_handler(each, admin.signalChildDied)
if hasattr(signal, 'SIGUSR1'):
set_signal_handler(signal.SIGUSR1, signal_admin_sts(admin))
_startLogger(transportClass, transport, admin, capabilities, logDefs,
concurrency_context)
#closeUnusedFiles(transport)
# Admin should never enter TX-only flow control state because this
# could deadlock or other non-progress conditions, especially if
# using admin routing.
transport.enableRXPauseFlowControl(False)
admin.run()
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass,
adminAddr, capabilities, logDefs):
# Unix Daemonization; skipped if not available
import os, sys
if hasattr(os, 'setsid'):
os.setsid()
try:
import resource
resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # No core dumps
except Exception:
pass
if hasattr(os, 'fork'):
if os.fork(): sys.exit(0)
# Slight trickiness here. There may *already* be an admin bound
# to this start address. However, the external process attempting
# to start is going to wait for the EndpointConnected message
# before continuing, so ensure that message is *not* sent until
# the local admin Transport has had time to bind and listen to the
# local address, but also ensure that the message is *always* sent
# even if the local admin could not start (caller will use
# _verifyAdminRunning to ensure things are OK.
transport = transportClass(endpointPrep)
try:
admin = adminClass(transport, adminAddr, capabilities, logDefs)
except Exception:
transport.scheduleTransmit(
None, TransmitIntent(addrOfStarter, EndpointConnected(0)))
raise
# Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below.
admin.addrOfStarter = addrOfStarter
setProcName(adminClass.__name__, admin.transport.myAddress)
# Generate the "placeholder" loggerAddr directly instead of going
# through the AddressManager because the logger is not managed as
# a normal child.
loggerAddr = ActorAddress(ActorLocalAddress(transport.myAddress, -1, None))
admin.asLogger = None
logAggregator = capabilities.get('Convention Address.IPv4', None)
if logAggregator:
try:
logAggregator = transportClass.getAddressFromString(logAggregator)
except Exception as ex:
thesplog(
'Unable to adapt log aggregator address "%s" to a transport address: %s',
logAggregator,
ex,
level=logging.WARNING)
admin.asLogProc = startASLogger(
loggerAddr, logDefs, transport, capabilities,
logAggregator if logAggregator != admin.transport.myAddress else None)
#closeUnusedFiles(transport)
admin.run()
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass,
adminAddr, capabilities, logDefs):
# Unix Daemonization; skipped if not available
import os,sys
if hasattr(os, 'setsid'):
os.setsid()
try:
import resource
resource.setrlimit(resource.RLIMIT_CORE, (0,0)) # No core dumps
except Exception: pass
if hasattr(os, 'fork'):
if os.fork(): sys.exit(0)
# Slight trickiness here. There may *already* be an admin bound
# to this start address. However, the external process attempting
# to start is going to wait for the EndpointConnected message
# before continuing, so ensure that message is *not* sent until
# the local admin Transport has had time to bind and listen to the
# local address, but also ensure that the message is *always* sent
# even if the local admin could not start (caller will use
# _verifyAdminRunning to ensure things are OK.
transport = transportClass(endpointPrep)
try:
admin = adminClass(transport, adminAddr, capabilities, logDefs)
except Exception:
transport.scheduleTransmit(None,
TransmitIntent(addrOfStarter, EndpointConnected(0)))
raise
# Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below.
admin.addrOfStarter = addrOfStarter
setProcName(adminClass.__name__, admin.transport.myAddress)
# Generate the "placeholder" loggerAddr directly instead of going
# through the AddressManager because the logger is not managed as
# a normal child.
loggerAddr = ActorAddress(ActorLocalAddress(transport.myAddress, -1, None))
admin.asLogger = None
logAggregator = capabilities.get('Convention Address.IPv4', None)
if logAggregator:
try:
logAggregator = transportClass.getAddressFromString(logAggregator)
except Exception as ex:
thesplog('Unable to adapt log aggregator address "%s" to a transport address: %s',
logAggregator, ex, level=logging.WARNING)
admin.asLogProc = startASLogger(loggerAddr, logDefs, transport, capabilities,
logAggregator
if logAggregator != admin.transport.myAddress
else None)
#closeUnusedFiles(transport)
admin.run()
def startChild(childClass, endpoint, transportClass,
sourceHash, sourceToLoad,
parentAddr, adminAddr, notifyAddr, loggerAddr,
childRequirements, currentSystemCapabilities,
fileNumsToClose):
closeFileNums(fileNumsToClose)
# Dirty trick here to workaround multiprocessing trying to impose
# an unnecessary restriction. A process should be set daemonic
# before start() if the parent shouldn't track it (an specifically
# automatically join() the subprocess on exit). For Actors, the
# parent exists independently of the child and the ActorSystem
# manages them, so daemonic processes are desired. However,
# multiprocessing imposes a restriction that daemonic processes
# cannot create more processes. The following reaches deep into
# the implementation of the multiprocessing module to override
# that restriction. This process was already started as daemonic,
# and it's detached from its parent. The following simply clears
# that flag locally so that other processes can be created from
# this one.
multiprocessing.process._current_process._daemonic = False
transport = transportClass(endpoint)
#closeUnusedFiles(transport)
# Dirty trick here to completely re-initialize logging in this
# process... something the standard Python logging interface does
# not allow via the API. We also do not want to run
# logging.shutdown() because (a) that does not do enough to reset,
# and (b) it shuts down handlers, but we want to leave the parent's
# handlers alone.
logging.root = ThespianLogForwarder(loggerAddr, transport)
logging.Logger.root = logging.root
logging.Logger.manager = logging.Manager(logging.Logger.root)
logger = logging.getLogger('Thespian.ActorManager')
am = MultiProcManager(childClass, transport,
sourceHash, sourceToLoad,
parentAddr, adminAddr,
childRequirements, currentSystemCapabilities)
am.asLogger = loggerAddr
am.transport.scheduleTransmit(None,
TransmitIntent(notifyAddr,
EndpointConnected(endpoint.addrInst)))
setProcName(getattr(childClass, '__name__', str(childClass)), am.transport.myAddress)
am.run()
def startChild(childClass, globalName, endpoint, transportClass, sourceHash,
sourceToLoad, parentAddr, adminAddr, notifyAddr, loggerAddr,
childRequirements, currentSystemCapabilities, fileNumsToClose,
concurrency_context):
closeFileNums(fileNumsToClose)
# Dirty trick here to workaround multiprocessing trying to impose
# an unnecessary restriction. A process should be set daemonic
# before start() if the parent shouldn't track it (an specifically
# automatically join() the subprocess on exit). For Actors, the
# parent exists independently of the child and the ActorSystem
# manages them, so daemonic processes are desired. However,
# multiprocessing imposes a restriction that daemonic processes
# cannot create more processes. The following reaches deep into
# the implementation of the multiprocessing module to override
# that restriction. This process was already started as daemonic,
# and it's detached from its parent. The following simply clears
# that flag locally so that other processes can be created from
# this one.
multiprocessing.process._current_process._daemonic = False
transport = transportClass(endpoint)
#closeUnusedFiles(transport)
# Dirty trick here to completely re-initialize logging in this
# process... something the standard Python logging interface does
# not allow via the API. We also do not want to run
# logging.shutdown() because (a) that does not do enough to reset,
# and (b) it shuts down handlers, but we want to leave the parent's
# handlers alone.
logging.root = ThespianLogForwarder(loggerAddr, transport)
logging.Logger.root = logging.root
logging.Logger.manager = logging.Manager(logging.Logger.root)
logger = logging.getLogger('Thespian.ActorManager')
am = MultiProcManager(childClass, globalName, transport, sourceHash,
sourceToLoad, parentAddr, adminAddr,
childRequirements, currentSystemCapabilities,
concurrency_context)
am.asLogger = loggerAddr
am.transport.scheduleTransmit(
None,
TransmitIntent(notifyAddr, EndpointConnected(
endpoint.addrInst)).addCallback(onFailure=am.actor_send_fail))
setProcName(
getattr(childClass, '__name__', str(childClass)).rpartition('.')[-1],
am.transport.myAddress)
sighandler = signal_detector(
getattr(childClass, '__name__', str(childClass)),
am.transport.myAddress, am)
sigexithandler = shutdown_signal_detector(
getattr(childClass, '__name__', str(childClass)),
am.transport.myAddress, am)
for each in range(1, signal.NSIG):
# n.b. normally Python intercepts SIGINT to turn it into a
# KeyboardInterrupt exception. However, these Actors should
# be detached from the keyboard, so revert to normal SIGINT
# behavior.
if each not in uncatchable_signals:
if each in child_exit_signals:
set_signal_handler(each, am.signalChildDied)
continue
try:
set_signal_handler(
each,
sigexithandler if each in exit_signals else sighandler)
except (RuntimeError, ValueError, EnvironmentError) as ex:
# OK, this signal can't be caught for this
# environment. We did our best.
pass
am.run()
def startupASLogger(addrOfStarter, logEndpoint, logDefs, transportClass,
aggregatorAddress):
# Dirty trick here to completely re-initialize logging in this
# process... something the standard Python logging interface does
# not allow via the API. We also do not want to run
# logging.shutdown() because (a) that does not do enough to reset,
# and (b) it shuts down handlers, but we want to leave the
# parent's handlers alone. Dirty trick here to completely
# re-initialize logging in this process... something the standard
# Python logging interface does not allow via the API.
logging.root = logging.RootLogger(logging.WARNING)
logging.Logger.root = logging.root
logging.Logger.manager = logging.Manager(logging.Logger.root)
if logDefs:
dictConfig(logDefs)
else:
logging.basicConfig()
# Disable thesplog from within the logging process (by setting the
# logfile size to zero) to try to avoid recursive logging loops.
thesplog_control(logging.WARNING, False, 0)
#logging.info('ActorSystem Logging Initialized')
transport = transportClass(logEndpoint)
setProcName('logger', transport.myAddress)
transport.scheduleTransmit(
None, TransmitIntent(addrOfStarter, LoggerConnected()))
fdup = None
last_exception_time = None
exception_count = 0
while True:
try:
r = transport.run(None)
if isinstance(r, Thespian__UpdateWork):
transport.scheduleTransmit(
TransmitIntent(transport.myAddress, r))
continue
logrecord = r.message
if isinstance(logrecord, LoggerExitRequest):
logging.info('ActorSystem Logging Shutdown')
return
elif isinstance(logrecord, LoggerFileDup):
fdup = getattr(logrecord, 'fname', None)
elif isinstance(logrecord, LogAggregator):
aggregatorAddress = logrecord.aggregatorAddress
elif isinstance(logrecord, logging.LogRecord):
logging.getLogger(logrecord.name).handle(logrecord)
if fdup:
with open(fdup, 'a') as ldf:
ldf.write('%s\n' % str(logrecord))
if aggregatorAddress and \
logrecord.levelno >= logging.WARNING:
transport.scheduleTransmit(
None, TransmitIntent(aggregatorAddress, logrecord))
else:
logging.warn('Unknown message rcvd by logger: %s' %
str(logrecord))
except Exception as ex:
thesplog('Thespian Logger aborting (#%d) with error %s',
exception_count,
ex,
exc_info=True)
if last_exception_time is None or \
last_exception_time.view().expired():
last_exception_time = ExpirationTimer(timedelta(seconds=1))
exception_count = 0
else:
exception_count += 1
if exception_count >= MAX_LOGGING_EXCEPTIONS_PER_SECOND:
thesplog(
'Too many Thespian Logger exceptions (#%d in %s); exiting!',
exception_count,
timedelta(seconds=1) -
last_exception_time.view().remaining())
return
def startChild(childClass, endpoint, transportClass,
sourceHash, sourceToLoad,
parentAddr, adminAddr, notifyAddr, loggerAddr,
childRequirements, currentSystemCapabilities,
fileNumsToClose, concurrency_context):
closeFileNums(fileNumsToClose)
# Dirty trick here to workaround multiprocessing trying to impose
# an unnecessary restriction. A process should be set daemonic
# before start() if the parent shouldn't track it (an specifically
# automatically join() the subprocess on exit). For Actors, the
# parent exists independently of the child and the ActorSystem
# manages them, so daemonic processes are desired. However,
# multiprocessing imposes a restriction that daemonic processes
# cannot create more processes. The following reaches deep into
# the implementation of the multiprocessing module to override
# that restriction. This process was already started as daemonic,
# and it's detached from its parent. The following simply clears
# that flag locally so that other processes can be created from
# this one.
multiprocessing.process._current_process._daemonic = False
transport = transportClass(endpoint)
#closeUnusedFiles(transport)
# Dirty trick here to completely re-initialize logging in this
# process... something the standard Python logging interface does
# not allow via the API. We also do not want to run
# logging.shutdown() because (a) that does not do enough to reset,
# and (b) it shuts down handlers, but we want to leave the parent's
# handlers alone.
logging.root = ThespianLogForwarder(loggerAddr, transport)
logging.Logger.root = logging.root
logging.Logger.manager = logging.Manager(logging.Logger.root)
logger = logging.getLogger('Thespian.ActorManager')
am = MultiProcManager(childClass, transport,
sourceHash, sourceToLoad,
parentAddr, adminAddr,
childRequirements, currentSystemCapabilities,
concurrency_context)
am.asLogger = loggerAddr
am.transport.scheduleTransmit(None,
TransmitIntent(notifyAddr,
EndpointConnected(endpoint.addrInst)))
setProcName(getattr(childClass, '__name__',
str(childClass)).rpartition('.')[-1],
am.transport.myAddress)
sighandler = signal_detector(getattr(childClass, '__name__', str(childClass)),
am.transport.myAddress, am)
sigexithandler = shutdown_signal_detector(getattr(childClass, '__name__', str(childClass)),
am.transport.myAddress,
am)
for each in range(1, signal.NSIG):
# n.b. normally Python intercepts SIGINT to turn it into a
# KeyboardInterrupt exception. However, these Actors should
# be detached from the keyboard, so revert to normal SIGINT
# behavior.
if each not in uncatchable_signals:
if each in child_exit_signals:
set_signal_handler(each, am.signalChildDied)
continue
try:
set_signal_handler(each,
sigexithandler if each in exit_signals
else sighandler)
except (RuntimeError,ValueError,EnvironmentError) as ex:
# OK, this signal can't be caught for this
# environment. We did our best.
pass
am.run()
