def disableMonitoringService(shutdown=False):
# disable the mon loop
log.debug("Shutting down the main monitoring loop")
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
log.debug("Disabling the central Monitoring")
from Ganga.Core import monitoring_component
monitoring_component.disableMonitoring()
if not shutdown:
from Ganga.GPI import queues
queues._purge_all()
first = 0
while queues.totalNumAllThreads() != 0:
log.debug("Ensuring that all tasks are purged from the todo!")
if first is not 0:
import time
time.sleep(0.5)
queues._purge_all()
queues._stop_all_threads()
from Ganga.Core.GangaThread.GangaThreadPool import GangaThreadPool
pool = GangaThreadPool.getInstance()
pool.shutdown()
pool.__do_shutdown__()
first = 1
log.debug("Queues Threads should now be gone")
def disableMonitoringService(shutdown=False):
# disable the mon loop
log.debug("Shutting down the main monitoring loop")
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
log.debug("Disabling the central Monitoring")
from Ganga.Core import monitoring_component
monitoring_component.disableMonitoring()
if not shutdown:
from Ganga.GPI import queues
queues._purge_all()
first = 0
while queues.totalNumAllThreads() != 0:
log.debug("Ensuring that all tasks are purged from the todo!")
if first is not 0:
import time
time.sleep(0.5)
queues._purge_all()
queues._stop_all_threads()
from Ganga.Core.GangaThread.GangaThreadPool import GangaThreadPool
pool = GangaThreadPool.getInstance()
pool.shutdown()
pool.__do_shutdown__()
first = 1
log.debug("Queues Threads should now be gone")
def __do_shutdown__(_all_threads):
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
from Ganga.GPI import queues
queues._purge_all()
queues._stop_all_threads(shutdown=True)
# while queues.totalNumAllThreads() != 0:
# queues._stop_all_threads()
# logger.warning( "Tasks still running: %s" % queues.threadStatus() )
# import time
# time.sleep( 0.1 )
logger.debug("ExternalTasks still running: %s" % queues.threadStatus())
logger.debug('Service threads to shutdown: %s' %
([i for i in reversed(list(_all_threads))]))
logger.debug('Service threads to shutdown: %s' %
([i for i in reversed(list(_all_threads))]))
# shutdown each individual threads in the pool
nonCritThreads = []
critThreads = []
for t in _all_threads:
if t.isCritical():
critThreads.append(t)
else:
nonCritThreads.append(t)
# while len( _all_threads ) != 0:
# Shutdown NON critical threads first as these can cause some critical
# threads to hang
for t in reversed(nonCritThreads):
logger.debug('shutting down Thread: %s' % t.getName())
t.stop()
logger.debug('shutdown Thread: %s' % t.getName())
# t.unregister()
# Shutdown critical threads now assuming that the non-critical ones
# have disappeared
for t in reversed(critThreads):
logger.debug('shutting down Thread: %s' % t.getName())
t.stop()
logger.debug('shutdown Thread: %s' % t.getName())
# t.unregister()
# nonCritThreads = []
# critThreads = []
# for t in _all_threads:
# if t.isCritical():
# critThreads.append( t )
# else:
# nonCritThreads.append( t )
def __cnt_alive_threads__(_all_threads):
num_alive_threads = 0
for t in _all_threads:
if t.isAlive():
num_alive_threads += 1
return num_alive_threads
num_alive_threads = __cnt_alive_threads__(_all_threads)
while num_alive_threads > 0:
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
# fix for bug #62543 https://savannah.cern.ch/bugs/?62543
# following 2 lines swapped so that we access no globals between
# sleep and exit test
num_alive_threads = __cnt_alive_threads__(_all_threads)
logger.debug('number of alive threads: %d' % num_alive_threads)
time.sleep(0.3)
num_alive_threads = __cnt_alive_threads__(_all_threads)
def _ganga_run_exitfuncs():
"""run any registered exit functions
atexit._exithandlers is traversed based on the priority.
If no priority was registered for a given function
than the lowest priority is assumed (LIFO policy)
We keep the same functionality as in *atexit* bare module but
we run each exit handler inside a try..catch block to be sure all
the registered handlers are executed
"""
#print("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.flush_all()
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'DiskIOTimeout', 1)
try:
from Ganga.GPI import queues
queues.lock()
except Exception as err:
logger.debug("This should only happen if Ganga filed to initialize correctly")
logger.debug("Err: %s" % str(err))
## Stop the Mon loop from iterating further!
from Ganga.Core import monitoring_component
if monitoring_component is not None:
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
getStackTrace()
monitoring_component.disableMonitoring()
#monitoring_component.stop()
## This will stop the Registries flat but we may still have threads processing data!
#from Ganga.Core.InternalServices import Coordinator
#if Coordinator.servicesEnabled:
# Coordinator.disableInternalServices( shutdown = True )
## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
from Ganga.GPIDev.Lib.Tasks import stopTasks
stopTasks()
# Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
#from Ganga.Utility.Config import setConfigOption
#setConfigOption('Configuration', 'DiskIOTimeout', 3)
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
try:
from Ganga.GPI import queues
queues._purge_all()
except Exception as err:
logger.debug("This should only happen if Ganga filed to initialize correctly")
logger.debug("Err2: %s" % str(err))
def priority_cmp(f1, f2):
"""
Sort the exit functions based on priority in reversed order
"""
# extract the priority number from the function element
p1 = f1[0][0]
p2 = f2[0][0]
# sort in reversed order
return cmp(p2, p1)
def add_priority(x):
"""
add a default priority to the functions not defining one (default priority=sys.maxint)
return a list containg ((priority,func),*targs,*kargs) elements
"""
import sys
func = x[0]
if isinstance(func, tuple) and len(x[0]) == 2:
return x
else:
new = [(sys.maxsize, func)]
new.extend(x[1:])
return new
atexit._exithandlers = map(add_priority, atexit._exithandlers)
atexit._exithandlers.sort(priority_cmp)
logger.info("Stopping running tasks before shutting down Repositories")
import inspect
while atexit._exithandlers:
(priority, func), targs, kargs = atexit._exithandlers.pop()
try:
if hasattr(func, 'im_class'):
for cls in inspect.getmro(func.__self__.__class__):
if func.__name__ in cls.__dict__:
logger.debug(cls.__name__ + " : " + func.__name__)
func(*targs, **kargs)
else:
logger.debug("noclass : " + func.__name__)
#print("%s" % str(func))
#print("%s" % str(inspect.getsourcefile(func)))
#func(*targs, **kargs)
## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
if str(inspect.getsourcefile(func)).find('Ganga') != -1:
func(*targs, **kargs)
except Exception as err:
s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (func.__name__, str(err))
logger.debug(s)
try:
import os
logger.debug("%s" % os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(func)))) )
logger.debug("\n%s" % inspect.getsource(func))
except Exception as err2:
logger.debug("Error getting source code and failure reason: %s" % str(err2))
logger.info("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
from Ganga.Core.InternalServices import Coordinator
Coordinator.servicesEnabled = False
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
import Ganga.Utility.logging
if Ganga.Utility.logging.requires_shutdown is True:
Ganga.Utility.logging.final_shutdown()
from Ganga.Runtime import bootstrap
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
def __do_shutdown__(_all_threads):
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
from Ganga.GPI import queues
queues._purge_all()
queues._stop_all_threads(shutdown=True)
# while queues.totalNumAllThreads() != 0:
# queues._stop_all_threads()
# logger.warning( "Tasks still running: %s" % queues.threadStatus() )
# import time
# time.sleep( 0.1 )
logger.debug("ExternalTasks still running: %s" % queues.threadStatus())
logger.debug('Service threads to shutdown: %s' % list(_all_threads))
logger.debug('Service threads to shutdown: %s' % list(_all_threads))
# shutdown each individual threads in the pool
nonCritThreads = []
critThreads = []
for t in _all_threads:
if t.isCritical():
critThreads.append(t)
else:
nonCritThreads.append(t)
# while len( _all_threads ) != 0:
# Shutdown NON critical threads first as these can cause some critical
# threads to hang
for t in reversed(nonCritThreads):
logger.debug('shutting down Thread: %s' % t.getName())
t.stop()
logger.debug('shutdown Thread: %s' % t.getName())
# t.unregister()
# Shutdown critical threads now assuming that the non-critical ones
# have disappeared
for t in reversed(critThreads):
logger.debug('shutting down Thread: %s' % t.getName())
t.stop()
logger.debug('shutdown Thread: %s' % t.getName())
# t.unregister()
# nonCritThreads = []
# critThreads = []
# for t in _all_threads:
# if t.isCritical():
# critThreads.append( t )
# else:
# nonCritThreads.append( t )
def __cnt_alive_threads__(_all_threads):
num_alive_threads = 0
for t in _all_threads:
if t.isAlive():
num_alive_threads += 1
return num_alive_threads
num_alive_threads = __cnt_alive_threads__(_all_threads)
while num_alive_threads > 0:
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
# fix for bug #62543 https://savannah.cern.ch/bugs/?62543
# following 2 lines swapped so that we access no globals between
# sleep and exit test
num_alive_threads = __cnt_alive_threads__(_all_threads)
logger.debug('number of alive threads: %d' % num_alive_threads)
time.sleep(0.3)
num_alive_threads = __cnt_alive_threads__(_all_threads)
def _ganga_run_exitfuncs():
"""run any registered exit functions
atexit._exithandlers is traversed based on the priority.
If no priority was registered for a given function
than the lowest priority is assumed (LIFO policy)
We keep the same functionality as in *atexit* bare module but
we run each exit handler inside a try..catch block to be sure all
the registered handlers are executed
"""
from Ganga.GPIDev.Base.Proxy import getName
#print("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.flush_all()
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'DiskIOTimeout', 1)
try:
from Ganga.GPI import queues
queues.lock()
except Exception as err:
logger.debug(
"This should only happen if Ganga filed to initialize correctly")
logger.debug("Err: %s" % str(err))
## Stop the Mon loop from iterating further!
from Ganga.Core import monitoring_component
if monitoring_component is not None:
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
getStackTrace()
monitoring_component.disableMonitoring()
#monitoring_component.stop()
## This will stop the Registries flat but we may still have threads processing data!
#from Ganga.Core.InternalServices import Coordinator
#if Coordinator.servicesEnabled:
# Coordinator.disableInternalServices( shutdown = True )
## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
from Ganga.GPIDev.Lib.Tasks import stopTasks
stopTasks()
# Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
#from Ganga.Utility.Config import setConfigOption
#setConfigOption('Configuration', 'DiskIOTimeout', 3)
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
try:
from Ganga.GPI import queues
queues._purge_all()
except Exception as err:
logger.debug(
"This should only happen if Ganga filed to initialize correctly")
logger.debug("Err2: %s" % str(err))
def priority_cmp(f1, f2):
"""
Sort the exit functions based on priority in reversed order
"""
# extract the priority number from the function element
p1 = f1[0][0]
p2 = f2[0][0]
# sort in reversed order
return cmp(p2, p1)
def add_priority(x):
"""
add a default priority to the functions not defining one (default priority=sys.maxint)
return a list containg ((priority,func),*targs,*kargs) elements
"""
import sys
func = x[0]
if isinstance(func, tuple) and len(x[0]) == 2:
return x
else:
new = [(sys.maxsize, func)]
new.extend(x[1:])
return new
atexit._exithandlers = map(add_priority, atexit._exithandlers)
atexit._exithandlers.sort(priority_cmp)
logger.info("Stopping running tasks before shutting down Repositories")
import inspect
while atexit._exithandlers:
(priority, func), targs, kargs = atexit._exithandlers.pop()
try:
if hasattr(func, 'im_class'):
for cls in inspect.getmro(func.__self__.__class__):
if getName(func) in cls.__dict__:
logger.debug(getName(cls) + " : " + getName(func))
func(*targs, **kargs)
else:
logger.debug("noclass : " + getName(func))
#print("%s" % str(func))
#print("%s" % str(inspect.getsourcefile(func)))
#func(*targs, **kargs)
## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
if str(inspect.getsourcefile(func)).find('Ganga') != -1:
func(*targs, **kargs)
except Exception as err:
s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (
getName(func), str(err))
logger.debug(s)
try:
import os
logger.debug("%s" % os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(func)))))
logger.debug("\n%s" % inspect.getsource(func))
except Exception as err2:
logger.debug(
"Error getting source code and failure reason: %s" %
str(err2))
logger.info("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
from Ganga.Core.InternalServices import Coordinator
Coordinator.servicesEnabled = False
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
from Ganga.Utility.logging import requires_shutdown, final_shutdown
if requires_shutdown is True:
final_shutdown()
from Ganga.Runtime import bootstrap
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
## This will stop the Registries flat but we may still have threads processing data!
#from Ganga.Core.InternalServices import Coordinator
#if Coordinator.servicesEnabled:
# Coordinator.disableInternalServices( shutdown = True )
# Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
#from Ganga.Utility.Config import setConfigOption
#setConfigOption('Configuration', 'DiskIOTimeout', 3)
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
try:
from Ganga.GPI import queues
queues._purge_all()
except Exception, err:
logger.debug("This should only happen if Ganga filed to initialize correctly")
def priority_cmp(f1, f2):
"""
Sort the exit functions based on priority in reversed order
"""
# extract the priority number from the function element
p1 = f1[0][0]
p2 = f2[0][0]
# sort in reversed order
return cmp(p2, p1)
def add_priority(x):
"""