def disableMonitoringService(shutdown=False): # disable the mon loop log.debug("Shutting down the main monitoring loop") from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool _purge_actions_queue() stop_and_free_thread_pool() log.debug("Disabling the central Monitoring") from Ganga.Core import monitoring_component monitoring_component.disableMonitoring() if not shutdown: from Ganga.GPI import queues queues._purge_all() first = 0 while queues.totalNumAllThreads() != 0: log.debug("Ensuring that all tasks are purged from the todo!") if first is not 0: import time time.sleep(0.5) queues._purge_all() queues._stop_all_threads() from Ganga.Core.GangaThread.GangaThreadPool import GangaThreadPool pool = GangaThreadPool.getInstance() pool.shutdown() pool.__do_shutdown__() first = 1 log.debug("Queues Threads should now be gone")
def __do_shutdown__(_all_threads): from Ganga.Utility.logging import getLogger logger = getLogger('GangaThread') from Ganga.GPI import queues queues._purge_all() queues._stop_all_threads(shutdown=True) # while queues.totalNumAllThreads() != 0: # queues._stop_all_threads() # logger.warning( "Tasks still running: %s" % queues.threadStatus() ) # import time # time.sleep( 0.1 ) logger.debug("ExternalTasks still running: %s" % queues.threadStatus()) logger.debug('Service threads to shutdown: %s' % ([i for i in reversed(list(_all_threads))])) logger.debug('Service threads to shutdown: %s' % ([i for i in reversed(list(_all_threads))])) # shutdown each individual threads in the pool nonCritThreads = [] critThreads = [] for t in _all_threads: if t.isCritical(): critThreads.append(t) else: nonCritThreads.append(t) # while len( _all_threads ) != 0: # Shutdown NON critical threads first as these can cause some critical # threads to hang for t in reversed(nonCritThreads): logger.debug('shutting down Thread: %s' % t.getName()) t.stop() logger.debug('shutdown Thread: %s' % t.getName()) # t.unregister() # Shutdown critical threads now assuming that the non-critical ones # have disappeared for t in reversed(critThreads): logger.debug('shutting down Thread: %s' % t.getName()) t.stop() logger.debug('shutdown Thread: %s' % t.getName()) # t.unregister() # nonCritThreads = [] # critThreads = [] # for t in _all_threads: # if t.isCritical(): # critThreads.append( t ) # else: # nonCritThreads.append( t ) def __cnt_alive_threads__(_all_threads): num_alive_threads = 0 for t in _all_threads: if t.isAlive(): num_alive_threads += 1 return num_alive_threads num_alive_threads = __cnt_alive_threads__(_all_threads) while num_alive_threads > 0: from Ganga.Utility.logging import getLogger logger = getLogger('GangaThread') # fix for bug #62543 https://savannah.cern.ch/bugs/?62543 # following 2 lines swapped so that we access no globals between # sleep and exit test num_alive_threads = __cnt_alive_threads__(_all_threads) logger.debug('number of alive threads: %d' % num_alive_threads) time.sleep(0.3) num_alive_threads = __cnt_alive_threads__(_all_threads)
def __do_shutdown__(_all_threads): from Ganga.Utility.logging import getLogger logger = getLogger('GangaThread') from Ganga.GPI import queues queues._purge_all() queues._stop_all_threads(shutdown=True) # while queues.totalNumAllThreads() != 0: # queues._stop_all_threads() # logger.warning( "Tasks still running: %s" % queues.threadStatus() ) # import time # time.sleep( 0.1 ) logger.debug("ExternalTasks still running: %s" % queues.threadStatus()) logger.debug('Service threads to shutdown: %s' % list(_all_threads)) logger.debug('Service threads to shutdown: %s' % list(_all_threads)) # shutdown each individual threads in the pool nonCritThreads = [] critThreads = [] for t in _all_threads: if t.isCritical(): critThreads.append(t) else: nonCritThreads.append(t) # while len( _all_threads ) != 0: # Shutdown NON critical threads first as these can cause some critical # threads to hang for t in reversed(nonCritThreads): logger.debug('shutting down Thread: %s' % t.getName()) t.stop() logger.debug('shutdown Thread: %s' % t.getName()) # t.unregister() # Shutdown critical threads now assuming that the non-critical ones # have disappeared for t in reversed(critThreads): logger.debug('shutting down Thread: %s' % t.getName()) t.stop() logger.debug('shutdown Thread: %s' % t.getName()) # t.unregister() # nonCritThreads = [] # critThreads = [] # for t in _all_threads: # if t.isCritical(): # critThreads.append( t ) # else: # nonCritThreads.append( t ) def __cnt_alive_threads__(_all_threads): num_alive_threads = 0 for t in _all_threads: if t.isAlive(): num_alive_threads += 1 return num_alive_threads num_alive_threads = __cnt_alive_threads__(_all_threads) while num_alive_threads > 0: from Ganga.Utility.logging import getLogger logger = getLogger('GangaThread') # fix for bug #62543 https://savannah.cern.ch/bugs/?62543 # following 2 lines swapped so that we access no globals between # sleep and exit test num_alive_threads = __cnt_alive_threads__(_all_threads) logger.debug('number of alive threads: %d' % num_alive_threads) time.sleep(0.3) num_alive_threads = __cnt_alive_threads__(_all_threads)
def shutdown(self, should_wait_cb=None): """Shutdown the Ganga session. @param should_wait_cb: A callback function with the following signature should_wait_cb(total_time, critical_thread_ids, non_critical_thread_ids) where total_time is the time in seconds since shutdown started critical_thread_ids is a list of alive critical thread names non_critical_thread_ids is a list of alive non-critical threads names. and return value is evaluated as a boolean. A shutdown thread is started that calls stop() on each GangaThread and waits for them all to die. A loop waits for the shutdown thread to die, periodically calling the should_wait_cb function to ask if it should continue to wait or shutdown anyway. """ from Ganga.GPI import queues queues._stop_all_threads() logger.debug( 'shutting down GangaThreadPool with timeout %d sec' % self.SHUTDOWN_TIMEOUT) # run shutdown thread in background import threading shutdown_thread = threading.Thread( target=self.__do_shutdown__, name='GANGA_Update_Thread_shutdown') shutdown_thread.setDaemon(True) shutdown_thread.start() t_start = time.time() # wait for the background shutdown thread to finish while shutdown_thread.isAlive(): logger.debug( 'Waiting for max %d seconds for threads to finish' % self.SHUTDOWN_TIMEOUT) logger.debug('There are %d alive background threads' % self.__cnt_alive_threads__()) logger.debug('%s' % self.__alive_critical_thread_ids()) logger.debug('%s' % self.__alive_non_critical_thread_ids()) shutdown_thread.join(self.SHUTDOWN_TIMEOUT) if shutdown_thread.isAlive(): # if should_wait_cb callback exists then ask if we should wait if should_wait_cb: total_time = time.time() - t_start critical_thread_ids = self.__alive_critical_thread_ids() non_critical_thread_ids = self.__alive_non_critical_thread_ids() if not should_wait_cb(total_time, critical_thread_ids, non_critical_thread_ids): logger.debug( 'GangaThreadPool shutdown anyway after %d sec.' % (time.time() - t_start)) break else: logger.debug('GangaThreadPool shutdown properly') # log warning message if critical thread still alive critical_thread_ids = self.__alive_critical_thread_ids() if critical_thread_ids: logger.warning('Shutdown forced. %d background thread(s) still running: %s', len( critical_thread_ids), critical_thread_ids) # log debug message if critical thread still alive non_critical_thread_ids = self.__alive_non_critical_thread_ids() if non_critical_thread_ids: logger.debug('Shutdown forced. %d non-critical background thread(s) still running: %s', len(non_critical_thread_ids), non_critical_thread_ids) # set singleton instance to None self._instance = None for i in self.__threads: del i self.__threads = []