def startupASLogger(addrOfStarter, logEndpoint, logDefs, transportClass, aggregatorAddress): # Dirty trick here to completely re-initialize logging in this # process... something the standard Python logging interface does # not allow via the API. We also do not want to run # logging.shutdown() because (a) that does not do enough to reset, # and (b) it shuts down handlers, but we want to leave the # parent's handlers alone. Dirty trick here to completely # re-initialize logging in this process... something the standard # Python logging interface does not allow via the API. logging.root = logging.RootLogger(logging.WARNING) logging.Logger.root = logging.root logging.Logger.manager = logging.Manager(logging.Logger.root) if logDefs: dictConfig(logDefs) else: logging.basicConfig() # Disable thesplog from within the logging process (by setting the # logfile size to zero) to try to avoid recursive logging loops. thesplog_control(logging.WARNING, False, 0) #logging.info('ActorSystem Logging Initialized') transport = transportClass(logEndpoint) setProcName('logger', transport.myAddress) transport.scheduleTransmit(None, TransmitIntent(addrOfStarter, LoggerConnected())) fdup = None last_exception = None last_exception_time = None exception_count = 0 while True: try: r = transport.run(None) logrecord = r.message if isinstance(logrecord, LoggerExitRequest): logging.info('ActorSystem Logging Shutdown') return elif isinstance(logrecord, LoggerFileDup): fdup = getattr(logrecord, 'fname', None) elif isinstance(logrecord, LogAggregator): aggregatorAddress = logrecord.aggregatorAddress elif isinstance(logrecord, logging.LogRecord): logging.getLogger(logrecord.name).handle(logrecord) if fdup: with open(fdup, 'a') as ldf: ldf.write('%s\n'%str(logrecord)) if aggregatorAddress and \ logrecord.levelno >= logging.WARNING: transport.scheduleTransmit(None, TransmitIntent(aggregatorAddress, logrecord)) else: logging.warn('Unknown message rcvd by logger: %s'%str(logrecord)) except Exception: logging.error('Thespian Logger aborting (#%d) with error', exception_count, exc_info=True) if last_exception is None or datetime.now() - last_exception_time > timedelta(seconds=1): last_exception_time = datetime.now() exception_count = 0 else: exception_count += 1 if exception_count >= MAX_LOGGING_EXCEPTIONS_PER_SECOND: logging.error('Too many Thespian Logger exceptions (#%d in %s); exiting!', exception_count, datetime.now() - last_exception_time) return
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass, adminAddr, capabilities, logDefs, concurrency_context): # Unix Daemonization; skipped if not available import os, sys if hasattr(os, 'setsid'): os.setsid() try: import resource resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # No core dumps except Exception: pass if hasattr(os, 'fork'): if os.fork(): sys.exit(0) # Slight trickiness here. There may *already* be an admin bound # to this start address. However, the external process attempting # to start is going to wait for the EndpointConnected message # before continuing, so ensure that message is *not* sent until # the local admin Transport has had time to bind and listen to the # local address, but also ensure that the message is *always* sent # even if the local admin could not start (caller will use # _verifyAdminRunning to ensure things are OK. transport = transportClass(endpointPrep) try: admin = adminClass(transport, adminAddr, capabilities, logDefs, concurrency_context) except Exception: transport.scheduleTransmit( None, TransmitIntent(addrOfStarter, EndpointConnected(0))) raise # Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below. admin.addrOfStarter = addrOfStarter setProcName( adminClass.__name__.rpartition('.')[-1], admin.transport.myAddress) # Admin does not do normal signal handling, but does want to know if children exit for each in range(1, signal.NSIG): # n.b. normally Python intercepts SIGINT to turn it into a # KeyboardInterrupt exception. However, these Actors should # be detached from the keyboard, so revert to normal SIGINT # behavior. if each not in uncatchable_signals: if each in child_exit_signals: set_signal_handler(each, admin.signalChildDied) if hasattr(signal, 'SIGUSR1'): set_signal_handler(signal.SIGUSR1, signal_admin_sts(admin)) _startLogger(transportClass, transport, admin, capabilities, logDefs, concurrency_context) #closeUnusedFiles(transport) # Admin should never enter TX-only flow control state because this # could deadlock or other non-progress conditions, especially if # using admin routing. transport.enableRXPauseFlowControl(False) admin.run()
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass, adminAddr, capabilities, logDefs, concurrency_context): # Unix Daemonization; skipped if not available import os,sys if hasattr(os, 'setsid'): os.setsid() try: import resource resource.setrlimit(resource.RLIMIT_CORE, (0,0)) # No core dumps except Exception: pass if hasattr(os, 'fork'): if os.fork(): sys.exit(0) # Slight trickiness here. There may *already* be an admin bound # to this start address. However, the external process attempting # to start is going to wait for the EndpointConnected message # before continuing, so ensure that message is *not* sent until # the local admin Transport has had time to bind and listen to the # local address, but also ensure that the message is *always* sent # even if the local admin could not start (caller will use # _verifyAdminRunning to ensure things are OK. transport = transportClass(endpointPrep) try: admin = adminClass(transport, adminAddr, capabilities, logDefs, concurrency_context) except Exception: transport.scheduleTransmit(None, TransmitIntent(addrOfStarter, EndpointConnected(0))) raise # Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below. admin.addrOfStarter = addrOfStarter setProcName(adminClass.__name__.rpartition('.')[-1], admin.transport.myAddress) # Admin does not do normal signal handling, but does want to know if children exit for each in range(1, signal.NSIG): # n.b. normally Python intercepts SIGINT to turn it into a # KeyboardInterrupt exception. However, these Actors should # be detached from the keyboard, so revert to normal SIGINT # behavior. if each not in uncatchable_signals: if each in child_exit_signals: set_signal_handler(each, admin.signalChildDied) if hasattr(signal, 'SIGUSR1'): set_signal_handler(signal.SIGUSR1, signal_admin_sts(admin)) _startLogger(transportClass, transport, admin, capabilities, logDefs, concurrency_context) #closeUnusedFiles(transport) # Admin should never enter TX-only flow control state because this # could deadlock or other non-progress conditions, especially if # using admin routing. transport.enableRXPauseFlowControl(False) admin.run()
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass, adminAddr, capabilities, logDefs): # Unix Daemonization; skipped if not available import os, sys if hasattr(os, 'setsid'): os.setsid() try: import resource resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # No core dumps except Exception: pass if hasattr(os, 'fork'): if os.fork(): sys.exit(0) # Slight trickiness here. There may *already* be an admin bound # to this start address. However, the external process attempting # to start is going to wait for the EndpointConnected message # before continuing, so ensure that message is *not* sent until # the local admin Transport has had time to bind and listen to the # local address, but also ensure that the message is *always* sent # even if the local admin could not start (caller will use # _verifyAdminRunning to ensure things are OK. transport = transportClass(endpointPrep) try: admin = adminClass(transport, adminAddr, capabilities, logDefs) except Exception: transport.scheduleTransmit( None, TransmitIntent(addrOfStarter, EndpointConnected(0))) raise # Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below. admin.addrOfStarter = addrOfStarter setProcName(adminClass.__name__, admin.transport.myAddress) # Generate the "placeholder" loggerAddr directly instead of going # through the AddressManager because the logger is not managed as # a normal child. loggerAddr = ActorAddress(ActorLocalAddress(transport.myAddress, -1, None)) admin.asLogger = None logAggregator = capabilities.get('Convention Address.IPv4', None) if logAggregator: try: logAggregator = transportClass.getAddressFromString(logAggregator) except Exception as ex: thesplog( 'Unable to adapt log aggregator address "%s" to a transport address: %s', logAggregator, ex, level=logging.WARNING) admin.asLogProc = startASLogger( loggerAddr, logDefs, transport, capabilities, logAggregator if logAggregator != admin.transport.myAddress else None) #closeUnusedFiles(transport) admin.run()
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass, adminAddr, capabilities, logDefs): # Unix Daemonization; skipped if not available import os,sys if hasattr(os, 'setsid'): os.setsid() try: import resource resource.setrlimit(resource.RLIMIT_CORE, (0,0)) # No core dumps except Exception: pass if hasattr(os, 'fork'): if os.fork(): sys.exit(0) # Slight trickiness here. There may *already* be an admin bound # to this start address. However, the external process attempting # to start is going to wait for the EndpointConnected message # before continuing, so ensure that message is *not* sent until # the local admin Transport has had time to bind and listen to the # local address, but also ensure that the message is *always* sent # even if the local admin could not start (caller will use # _verifyAdminRunning to ensure things are OK. transport = transportClass(endpointPrep) try: admin = adminClass(transport, adminAddr, capabilities, logDefs) except Exception: transport.scheduleTransmit(None, TransmitIntent(addrOfStarter, EndpointConnected(0))) raise # Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below. admin.addrOfStarter = addrOfStarter setProcName(adminClass.__name__, admin.transport.myAddress) # Generate the "placeholder" loggerAddr directly instead of going # through the AddressManager because the logger is not managed as # a normal child. loggerAddr = ActorAddress(ActorLocalAddress(transport.myAddress, -1, None)) admin.asLogger = None logAggregator = capabilities.get('Convention Address.IPv4', None) if logAggregator: try: logAggregator = transportClass.getAddressFromString(logAggregator) except Exception as ex: thesplog('Unable to adapt log aggregator address "%s" to a transport address: %s', logAggregator, ex, level=logging.WARNING) admin.asLogProc = startASLogger(loggerAddr, logDefs, transport, capabilities, logAggregator if logAggregator != admin.transport.myAddress else None) #closeUnusedFiles(transport) admin.run()
def startChild(childClass, endpoint, transportClass, sourceHash, sourceToLoad, parentAddr, adminAddr, notifyAddr, loggerAddr, childRequirements, currentSystemCapabilities, fileNumsToClose): closeFileNums(fileNumsToClose) # Dirty trick here to workaround multiprocessing trying to impose # an unnecessary restriction. A process should be set daemonic # before start() if the parent shouldn't track it (an specifically # automatically join() the subprocess on exit). For Actors, the # parent exists independently of the child and the ActorSystem # manages them, so daemonic processes are desired. However, # multiprocessing imposes a restriction that daemonic processes # cannot create more processes. The following reaches deep into # the implementation of the multiprocessing module to override # that restriction. This process was already started as daemonic, # and it's detached from its parent. The following simply clears # that flag locally so that other processes can be created from # this one. multiprocessing.process._current_process._daemonic = False transport = transportClass(endpoint) #closeUnusedFiles(transport) # Dirty trick here to completely re-initialize logging in this # process... something the standard Python logging interface does # not allow via the API. We also do not want to run # logging.shutdown() because (a) that does not do enough to reset, # and (b) it shuts down handlers, but we want to leave the parent's # handlers alone. logging.root = ThespianLogForwarder(loggerAddr, transport) logging.Logger.root = logging.root logging.Logger.manager = logging.Manager(logging.Logger.root) logger = logging.getLogger('Thespian.ActorManager') am = MultiProcManager(childClass, transport, sourceHash, sourceToLoad, parentAddr, adminAddr, childRequirements, currentSystemCapabilities) am.asLogger = loggerAddr am.transport.scheduleTransmit(None, TransmitIntent(notifyAddr, EndpointConnected(endpoint.addrInst))) setProcName(getattr(childClass, '__name__', str(childClass)), am.transport.myAddress) am.run()
def startChild(childClass, globalName, endpoint, transportClass, sourceHash, sourceToLoad, parentAddr, adminAddr, notifyAddr, loggerAddr, childRequirements, currentSystemCapabilities, fileNumsToClose, concurrency_context): closeFileNums(fileNumsToClose) # Dirty trick here to workaround multiprocessing trying to impose # an unnecessary restriction. A process should be set daemonic # before start() if the parent shouldn't track it (an specifically # automatically join() the subprocess on exit). For Actors, the # parent exists independently of the child and the ActorSystem # manages them, so daemonic processes are desired. However, # multiprocessing imposes a restriction that daemonic processes # cannot create more processes. The following reaches deep into # the implementation of the multiprocessing module to override # that restriction. This process was already started as daemonic, # and it's detached from its parent. The following simply clears # that flag locally so that other processes can be created from # this one. multiprocessing.process._current_process._daemonic = False transport = transportClass(endpoint) #closeUnusedFiles(transport) # Dirty trick here to completely re-initialize logging in this # process... something the standard Python logging interface does # not allow via the API. We also do not want to run # logging.shutdown() because (a) that does not do enough to reset, # and (b) it shuts down handlers, but we want to leave the parent's # handlers alone. logging.root = ThespianLogForwarder(loggerAddr, transport) logging.Logger.root = logging.root logging.Logger.manager = logging.Manager(logging.Logger.root) logger = logging.getLogger('Thespian.ActorManager') am = MultiProcManager(childClass, globalName, transport, sourceHash, sourceToLoad, parentAddr, adminAddr, childRequirements, currentSystemCapabilities, concurrency_context) am.asLogger = loggerAddr am.transport.scheduleTransmit( None, TransmitIntent(notifyAddr, EndpointConnected( endpoint.addrInst)).addCallback(onFailure=am.actor_send_fail)) setProcName( getattr(childClass, '__name__', str(childClass)).rpartition('.')[-1], am.transport.myAddress) sighandler = signal_detector( getattr(childClass, '__name__', str(childClass)), am.transport.myAddress, am) sigexithandler = shutdown_signal_detector( getattr(childClass, '__name__', str(childClass)), am.transport.myAddress, am) for each in range(1, signal.NSIG): # n.b. normally Python intercepts SIGINT to turn it into a # KeyboardInterrupt exception. However, these Actors should # be detached from the keyboard, so revert to normal SIGINT # behavior. if each not in uncatchable_signals: if each in child_exit_signals: set_signal_handler(each, am.signalChildDied) continue try: set_signal_handler( each, sigexithandler if each in exit_signals else sighandler) except (RuntimeError, ValueError, EnvironmentError) as ex: # OK, this signal can't be caught for this # environment. We did our best. pass am.run()
def startupASLogger(addrOfStarter, logEndpoint, logDefs, transportClass, aggregatorAddress): # Dirty trick here to completely re-initialize logging in this # process... something the standard Python logging interface does # not allow via the API. We also do not want to run # logging.shutdown() because (a) that does not do enough to reset, # and (b) it shuts down handlers, but we want to leave the # parent's handlers alone. Dirty trick here to completely # re-initialize logging in this process... something the standard # Python logging interface does not allow via the API. logging.root = logging.RootLogger(logging.WARNING) logging.Logger.root = logging.root logging.Logger.manager = logging.Manager(logging.Logger.root) if logDefs: dictConfig(logDefs) else: logging.basicConfig() # Disable thesplog from within the logging process (by setting the # logfile size to zero) to try to avoid recursive logging loops. thesplog_control(logging.WARNING, False, 0) #logging.info('ActorSystem Logging Initialized') transport = transportClass(logEndpoint) setProcName('logger', transport.myAddress) transport.scheduleTransmit( None, TransmitIntent(addrOfStarter, LoggerConnected())) fdup = None last_exception_time = None exception_count = 0 while True: try: r = transport.run(None) if isinstance(r, Thespian__UpdateWork): transport.scheduleTransmit( TransmitIntent(transport.myAddress, r)) continue logrecord = r.message if isinstance(logrecord, LoggerExitRequest): logging.info('ActorSystem Logging Shutdown') return elif isinstance(logrecord, LoggerFileDup): fdup = getattr(logrecord, 'fname', None) elif isinstance(logrecord, LogAggregator): aggregatorAddress = logrecord.aggregatorAddress elif isinstance(logrecord, logging.LogRecord): logging.getLogger(logrecord.name).handle(logrecord) if fdup: with open(fdup, 'a') as ldf: ldf.write('%s\n' % str(logrecord)) if aggregatorAddress and \ logrecord.levelno >= logging.WARNING: transport.scheduleTransmit( None, TransmitIntent(aggregatorAddress, logrecord)) else: logging.warn('Unknown message rcvd by logger: %s' % str(logrecord)) except Exception as ex: thesplog('Thespian Logger aborting (#%d) with error %s', exception_count, ex, exc_info=True) if last_exception_time is None or \ last_exception_time.view().expired(): last_exception_time = ExpirationTimer(timedelta(seconds=1)) exception_count = 0 else: exception_count += 1 if exception_count >= MAX_LOGGING_EXCEPTIONS_PER_SECOND: thesplog( 'Too many Thespian Logger exceptions (#%d in %s); exiting!', exception_count, timedelta(seconds=1) - last_exception_time.view().remaining()) return
def startChild(childClass, endpoint, transportClass, sourceHash, sourceToLoad, parentAddr, adminAddr, notifyAddr, loggerAddr, childRequirements, currentSystemCapabilities, fileNumsToClose, concurrency_context): closeFileNums(fileNumsToClose) # Dirty trick here to workaround multiprocessing trying to impose # an unnecessary restriction. A process should be set daemonic # before start() if the parent shouldn't track it (an specifically # automatically join() the subprocess on exit). For Actors, the # parent exists independently of the child and the ActorSystem # manages them, so daemonic processes are desired. However, # multiprocessing imposes a restriction that daemonic processes # cannot create more processes. The following reaches deep into # the implementation of the multiprocessing module to override # that restriction. This process was already started as daemonic, # and it's detached from its parent. The following simply clears # that flag locally so that other processes can be created from # this one. multiprocessing.process._current_process._daemonic = False transport = transportClass(endpoint) #closeUnusedFiles(transport) # Dirty trick here to completely re-initialize logging in this # process... something the standard Python logging interface does # not allow via the API. We also do not want to run # logging.shutdown() because (a) that does not do enough to reset, # and (b) it shuts down handlers, but we want to leave the parent's # handlers alone. logging.root = ThespianLogForwarder(loggerAddr, transport) logging.Logger.root = logging.root logging.Logger.manager = logging.Manager(logging.Logger.root) logger = logging.getLogger('Thespian.ActorManager') am = MultiProcManager(childClass, transport, sourceHash, sourceToLoad, parentAddr, adminAddr, childRequirements, currentSystemCapabilities, concurrency_context) am.asLogger = loggerAddr am.transport.scheduleTransmit(None, TransmitIntent(notifyAddr, EndpointConnected(endpoint.addrInst))) setProcName(getattr(childClass, '__name__', str(childClass)).rpartition('.')[-1], am.transport.myAddress) sighandler = signal_detector(getattr(childClass, '__name__', str(childClass)), am.transport.myAddress, am) sigexithandler = shutdown_signal_detector(getattr(childClass, '__name__', str(childClass)), am.transport.myAddress, am) for each in range(1, signal.NSIG): # n.b. normally Python intercepts SIGINT to turn it into a # KeyboardInterrupt exception. However, these Actors should # be detached from the keyboard, so revert to normal SIGINT # behavior. if each not in uncatchable_signals: if each in child_exit_signals: set_signal_handler(each, am.signalChildDied) continue try: set_signal_handler(each, sigexithandler if each in exit_signals else sighandler) except (RuntimeError,ValueError,EnvironmentError) as ex: # OK, this signal can't be caught for this # environment. We did our best. pass am.run()