def disableMonitoringService(shutdown=False):
# disable the mon loop
log.debug("Shutting down the main monitoring loop")
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
log.debug("Disabling the central Monitoring")
from Ganga.Core import monitoring_component
monitoring_component.disableMonitoring()
if not shutdown:
from Ganga.GPI import queues
queues._purge_all()
first = 0
while queues.totalNumAllThreads() != 0:
log.debug("Ensuring that all tasks are purged from the todo!")
if first is not 0:
import time
time.sleep(0.5)
queues._purge_all()
queues._stop_all_threads()
from Ganga.Core.GangaThread.GangaThreadPool import GangaThreadPool
pool = GangaThreadPool.getInstance()
pool.shutdown()
pool.__do_shutdown__()
first = 1
log.debug("Queues Threads should now be gone")
def disableMonitoringService(shutdown=False):
# disable the mon loop
log.debug("Shutting down the main monitoring loop")
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
log.debug("Disabling the central Monitoring")
from Ganga.Core import monitoring_component
monitoring_component.disableMonitoring()
if not shutdown:
from Ganga.GPI import queues
queues._purge_all()
first = 0
while queues.totalNumAllThreads() != 0:
log.debug("Ensuring that all tasks are purged from the todo!")
if first is not 0:
import time
time.sleep(0.5)
queues._purge_all()
queues._stop_all_threads()
from Ganga.Core.GangaThread.GangaThreadPool import GangaThreadPool
pool = GangaThreadPool.getInstance()
pool.shutdown()
pool.__do_shutdown__()
first = 1
log.debug("Queues Threads should now be gone")
def __do_shutdown__(_all_threads):
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
from Ganga.GPI import queues
queues._purge_all()
queues._stop_all_threads(shutdown=True)
# while queues.totalNumAllThreads() != 0:
# queues._stop_all_threads()
# logger.warning( "Tasks still running: %s" % queues.threadStatus() )
# import time
# time.sleep( 0.1 )
logger.debug("ExternalTasks still running: %s" % queues.threadStatus())
logger.debug('Service threads to shutdown: %s' %
([i for i in reversed(list(_all_threads))]))
logger.debug('Service threads to shutdown: %s' %
([i for i in reversed(list(_all_threads))]))
# shutdown each individual threads in the pool
nonCritThreads = []
critThreads = []
for t in _all_threads:
if t.isCritical():
critThreads.append(t)
else:
nonCritThreads.append(t)
# while len( _all_threads ) != 0:
# Shutdown NON critical threads first as these can cause some critical
# threads to hang
for t in reversed(nonCritThreads):
logger.debug('shutting down Thread: %s' % t.getName())
t.stop()
logger.debug('shutdown Thread: %s' % t.getName())
# t.unregister()
# Shutdown critical threads now assuming that the non-critical ones
# have disappeared
for t in reversed(critThreads):
logger.debug('shutting down Thread: %s' % t.getName())
t.stop()
logger.debug('shutdown Thread: %s' % t.getName())
# t.unregister()
# nonCritThreads = []
# critThreads = []
# for t in _all_threads:
# if t.isCritical():
# critThreads.append( t )
# else:
# nonCritThreads.append( t )
def __cnt_alive_threads__(_all_threads):
num_alive_threads = 0
for t in _all_threads:
if t.isAlive():
num_alive_threads += 1
return num_alive_threads
num_alive_threads = __cnt_alive_threads__(_all_threads)
while num_alive_threads > 0:
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
# fix for bug #62543 https://savannah.cern.ch/bugs/?62543
# following 2 lines swapped so that we access no globals between
# sleep and exit test
num_alive_threads = __cnt_alive_threads__(_all_threads)
logger.debug('number of alive threads: %d' % num_alive_threads)
time.sleep(0.3)
num_alive_threads = __cnt_alive_threads__(_all_threads)
def __do_shutdown__(_all_threads):
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
from Ganga.GPI import queues
queues._purge_all()
queues._stop_all_threads(shutdown=True)
# while queues.totalNumAllThreads() != 0:
# queues._stop_all_threads()
# logger.warning( "Tasks still running: %s" % queues.threadStatus() )
# import time
# time.sleep( 0.1 )
logger.debug("ExternalTasks still running: %s" % queues.threadStatus())
logger.debug('Service threads to shutdown: %s' % list(_all_threads))
logger.debug('Service threads to shutdown: %s' % list(_all_threads))
# shutdown each individual threads in the pool
nonCritThreads = []
critThreads = []
for t in _all_threads:
if t.isCritical():
critThreads.append(t)
else:
nonCritThreads.append(t)
# while len( _all_threads ) != 0:
# Shutdown NON critical threads first as these can cause some critical
# threads to hang
for t in reversed(nonCritThreads):
logger.debug('shutting down Thread: %s' % t.getName())
t.stop()
logger.debug('shutdown Thread: %s' % t.getName())
# t.unregister()
# Shutdown critical threads now assuming that the non-critical ones
# have disappeared
for t in reversed(critThreads):
logger.debug('shutting down Thread: %s' % t.getName())
t.stop()
logger.debug('shutdown Thread: %s' % t.getName())
# t.unregister()
# nonCritThreads = []
# critThreads = []
# for t in _all_threads:
# if t.isCritical():
# critThreads.append( t )
# else:
# nonCritThreads.append( t )
def __cnt_alive_threads__(_all_threads):
num_alive_threads = 0
for t in _all_threads:
if t.isAlive():
num_alive_threads += 1
return num_alive_threads
num_alive_threads = __cnt_alive_threads__(_all_threads)
while num_alive_threads > 0:
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
# fix for bug #62543 https://savannah.cern.ch/bugs/?62543
# following 2 lines swapped so that we access no globals between
# sleep and exit test
num_alive_threads = __cnt_alive_threads__(_all_threads)
logger.debug('number of alive threads: %d' % num_alive_threads)
time.sleep(0.3)
num_alive_threads = __cnt_alive_threads__(_all_threads)
def shutdown(self, should_wait_cb=None):
"""Shutdown the Ganga session.
@param should_wait_cb: A callback function with the following signature
should_wait_cb(total_time, critical_thread_ids, non_critical_thread_ids)
where
total_time is the time in seconds since shutdown started
critical_thread_ids is a list of alive critical thread names
non_critical_thread_ids is a list of alive non-critical threads names.
and
return value is evaluated as a boolean.
A shutdown thread is started that calls stop() on each GangaThread and
waits for them all to die. A loop waits for the shutdown thread to
die, periodically calling the should_wait_cb function to ask if it
should continue to wait or shutdown anyway.
"""
from Ganga.GPI import queues
queues._stop_all_threads()
logger.debug(
'shutting down GangaThreadPool with timeout %d sec' % self.SHUTDOWN_TIMEOUT)
# run shutdown thread in background
import threading
shutdown_thread = threading.Thread(
target=self.__do_shutdown__, name='GANGA_Update_Thread_shutdown')
shutdown_thread.setDaemon(True)
shutdown_thread.start()
t_start = time.time()
# wait for the background shutdown thread to finish
while shutdown_thread.isAlive():
logger.debug(
'Waiting for max %d seconds for threads to finish' % self.SHUTDOWN_TIMEOUT)
logger.debug('There are %d alive background threads' %
self.__cnt_alive_threads__())
logger.debug('%s' % self.__alive_critical_thread_ids())
logger.debug('%s' % self.__alive_non_critical_thread_ids())
shutdown_thread.join(self.SHUTDOWN_TIMEOUT)
if shutdown_thread.isAlive():
# if should_wait_cb callback exists then ask if we should wait
if should_wait_cb:
total_time = time.time() - t_start
critical_thread_ids = self.__alive_critical_thread_ids()
non_critical_thread_ids = self.__alive_non_critical_thread_ids()
if not should_wait_cb(total_time, critical_thread_ids, non_critical_thread_ids):
logger.debug(
'GangaThreadPool shutdown anyway after %d sec.' % (time.time() - t_start))
break
else:
logger.debug('GangaThreadPool shutdown properly')
# log warning message if critical thread still alive
critical_thread_ids = self.__alive_critical_thread_ids()
if critical_thread_ids:
logger.warning('Shutdown forced. %d background thread(s) still running: %s', len(
critical_thread_ids), critical_thread_ids)
# log debug message if critical thread still alive
non_critical_thread_ids = self.__alive_non_critical_thread_ids()
if non_critical_thread_ids:
logger.debug('Shutdown forced. %d non-critical background thread(s) still running: %s',
len(non_critical_thread_ids), non_critical_thread_ids)
# set singleton instance to None
self._instance = None
for i in self.__threads:
del i
self.__threads = []
