Пример #1
0
def disableMonitoringService(shutdown=False):

    # disable the mon loop
    log.debug("Shutting down the main monitoring loop")
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool

    _purge_actions_queue()
    stop_and_free_thread_pool()
    log.debug("Disabling the central Monitoring")
    from Ganga.Core import monitoring_component

    monitoring_component.disableMonitoring()

    if not shutdown:
        from Ganga.GPI import queues

        queues._purge_all()
        first = 0
        while queues.totalNumAllThreads() != 0:
            log.debug("Ensuring that all tasks are purged from the todo!")
            if first is not 0:
                import time

                time.sleep(0.5)
            queues._purge_all()
            queues._stop_all_threads()
            from Ganga.Core.GangaThread.GangaThreadPool import GangaThreadPool

            pool = GangaThreadPool.getInstance()
            pool.shutdown()
            pool.__do_shutdown__()
            first = 1

        log.debug("Queues Threads should now be gone")
Пример #2
0
def disableMonitoringService(shutdown=False):

    # disable the mon loop
    log.debug("Shutting down the main monitoring loop")
    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
    _purge_actions_queue()
    stop_and_free_thread_pool()
    log.debug("Disabling the central Monitoring")
    from Ganga.Core import monitoring_component
    monitoring_component.disableMonitoring()

    if not shutdown:
        from Ganga.GPI import queues
        queues._purge_all()
        first = 0
        while queues.totalNumAllThreads() != 0:
            log.debug("Ensuring that all tasks are purged from the todo!")
            if first is not 0:
                import time
                time.sleep(0.5)
            queues._purge_all()
            queues._stop_all_threads()
            from Ganga.Core.GangaThread.GangaThreadPool import GangaThreadPool
            pool = GangaThreadPool.getInstance()
            pool.shutdown()
            pool.__do_shutdown__()
            first = 1
        
        log.debug("Queues Threads should now be gone")
Пример #3
0
    def __do_shutdown__(_all_threads):

        from Ganga.Utility.logging import getLogger
        logger = getLogger('GangaThread')

        from Ganga.GPI import queues

        queues._purge_all()
        queues._stop_all_threads(shutdown=True)

        # while queues.totalNumAllThreads() != 0:
        #    queues._stop_all_threads()
        #    logger.warning( "Tasks still running: %s" % queues.threadStatus() )
        #    import time
        #    time.sleep( 0.1 )

        logger.debug("ExternalTasks still running: %s" % queues.threadStatus())

        logger.debug('Service threads to shutdown: %s' %
                     ([i for i in reversed(list(_all_threads))]))

        logger.debug('Service threads to shutdown: %s' %
                     ([i for i in reversed(list(_all_threads))]))

        # shutdown each individual threads in the pool
        nonCritThreads = []
        critThreads = []

        for t in _all_threads:
            if t.isCritical():
                critThreads.append(t)
            else:
                nonCritThreads.append(t)

        # while len( _all_threads ) != 0:
        # Shutdown NON critical threads first as these can cause some critical
        # threads to hang
        for t in reversed(nonCritThreads):
            logger.debug('shutting down Thread: %s' % t.getName())
            t.stop()
            logger.debug('shutdown Thread: %s' % t.getName())
            # t.unregister()

        # Shutdown critical threads now assuming that the non-critical ones
        # have disappeared
        for t in reversed(critThreads):
            logger.debug('shutting down Thread: %s' % t.getName())
            t.stop()
            logger.debug('shutdown Thread: %s' % t.getName())
            # t.unregister()

        #    nonCritThreads = []
        #    critThreads = []

        #    for t in _all_threads:
        #        if t.isCritical():
        #            critThreads.append( t )
        #        else:
        #            nonCritThreads.append( t )

        def __cnt_alive_threads__(_all_threads):
            num_alive_threads = 0
            for t in _all_threads:
                if t.isAlive():
                    num_alive_threads += 1
            return num_alive_threads

        num_alive_threads = __cnt_alive_threads__(_all_threads)

        while num_alive_threads > 0:
            from Ganga.Utility.logging import getLogger
            logger = getLogger('GangaThread')
            # fix for bug #62543 https://savannah.cern.ch/bugs/?62543
            # following 2 lines swapped so that we access no globals between
            # sleep and exit test
            num_alive_threads = __cnt_alive_threads__(_all_threads)
            logger.debug('number of alive threads: %d' % num_alive_threads)
            time.sleep(0.3)
            num_alive_threads = __cnt_alive_threads__(_all_threads)
Пример #4
0
def _ganga_run_exitfuncs():
    """run any registered exit functions

    atexit._exithandlers is traversed based on the priority.
    If no priority was registered for a given function
    than the lowest priority is assumed (LIFO policy)

    We keep the same functionality as in *atexit* bare module but
    we run each exit handler inside a try..catch block to be sure all
    the registered handlers are executed
    """

    #print("Shutting Down Ganga Repositories")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.flush_all()

    from Ganga.Utility.logging import getLogger
    logger = getLogger()

    # Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
    from Ganga.Utility.Config import setConfigOption
    setConfigOption('Configuration', 'DiskIOTimeout', 1)

    try:
        from Ganga.GPI import queues
        queues.lock()
    except Exception as err:
        logger.debug("This should only happen if Ganga filed to initialize correctly")
        logger.debug("Err: %s" % str(err))

    ## Stop the Mon loop from iterating further!
    from Ganga.Core import monitoring_component
    if monitoring_component is not None:
        from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
        getStackTrace()
        monitoring_component.disableMonitoring()
        #monitoring_component.stop()

    ## This will stop the Registries flat but we may still have threads processing data!
    #from Ganga.Core.InternalServices import Coordinator
    #if Coordinator.servicesEnabled:
    #    Coordinator.disableInternalServices( shutdown = True )

    ## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
    from Ganga.GPIDev.Lib.Tasks import stopTasks
    stopTasks()

    # Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
    #from Ganga.Utility.Config import setConfigOption
    #setConfigOption('Configuration', 'DiskIOTimeout', 3)

    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
    _purge_actions_queue()
    stop_and_free_thread_pool()

    try:
        from Ganga.GPI import queues
        queues._purge_all()
    except Exception as err:
        logger.debug("This should only happen if Ganga filed to initialize correctly")
        logger.debug("Err2: %s" % str(err))

    def priority_cmp(f1, f2):
        """
        Sort the exit functions based on priority in reversed order
        """
        # extract the priority number from the function element
        p1 = f1[0][0]
        p2 = f2[0][0]
        # sort in reversed order
        return cmp(p2, p1)

    def add_priority(x):
        """
        add a default priority to the functions not defining one (default priority=sys.maxint)
        return a list containg ((priority,func),*targs,*kargs) elements
        """
        import sys
        func = x[0]
        if isinstance(func, tuple) and len(x[0]) == 2:
            return x
        else:
            new = [(sys.maxsize, func)]
            new.extend(x[1:])
            return new

    atexit._exithandlers = map(add_priority, atexit._exithandlers)
    atexit._exithandlers.sort(priority_cmp)

    logger.info("Stopping running tasks before shutting down Repositories")

    import inspect
    while atexit._exithandlers:

        (priority, func), targs, kargs = atexit._exithandlers.pop()
        try:
            if hasattr(func, 'im_class'):
                for cls in inspect.getmro(func.__self__.__class__):
                    if func.__name__ in cls.__dict__:
                        logger.debug(cls.__name__ + " : " + func.__name__)
                func(*targs, **kargs)
            else:
                logger.debug("noclass : " + func.__name__)
                #print("%s" % str(func))
                #print("%s" % str(inspect.getsourcefile(func)))
                #func(*targs, **kargs)
                ## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
                if str(inspect.getsourcefile(func)).find('Ganga') != -1:
                    func(*targs, **kargs)
        except Exception as err:
            s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (func.__name__, str(err))
            logger.debug(s)
            try:
                import os
                logger.debug("%s" % os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(func)))) )
                logger.debug("\n%s" % inspect.getsource(func))
            except Exception as err2:
                logger.debug("Error getting source code and failure reason: %s" % str(err2))

    logger.info("Shutting Down Ganga Repositories")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.shutdown()

    from Ganga.Core.InternalServices import Coordinator
    Coordinator.servicesEnabled = False

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    import Ganga.Utility.logging
    if Ganga.Utility.logging.requires_shutdown is True:
        Ganga.Utility.logging.final_shutdown()

    from Ganga.Runtime import bootstrap
    if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
        bootstrap.printOpenFiles()
Пример #5
0
    def __do_shutdown__(_all_threads):

        from Ganga.Utility.logging import getLogger
        logger = getLogger('GangaThread')

        from Ganga.GPI import queues

        queues._purge_all()
        queues._stop_all_threads(shutdown=True)

        # while queues.totalNumAllThreads() != 0:
        #    queues._stop_all_threads()
        #    logger.warning( "Tasks still running: %s" % queues.threadStatus() )
        #    import time
        #    time.sleep( 0.1 )

        logger.debug("ExternalTasks still running: %s" % queues.threadStatus())

        logger.debug('Service threads to shutdown: %s' % list(_all_threads))

        logger.debug('Service threads to shutdown: %s' % list(_all_threads))

        # shutdown each individual threads in the pool
        nonCritThreads = []
        critThreads = []

        for t in _all_threads:
            if t.isCritical():
                critThreads.append(t)
            else:
                nonCritThreads.append(t)

        # while len( _all_threads ) != 0:
        # Shutdown NON critical threads first as these can cause some critical
        # threads to hang
        for t in reversed(nonCritThreads):
            logger.debug('shutting down Thread: %s' % t.getName())
            t.stop()
            logger.debug('shutdown Thread: %s' % t.getName())
            # t.unregister()

        # Shutdown critical threads now assuming that the non-critical ones
        # have disappeared
        for t in reversed(critThreads):
            logger.debug('shutting down Thread: %s' % t.getName())
            t.stop()
            logger.debug('shutdown Thread: %s' % t.getName())
            # t.unregister()

        #    nonCritThreads = []
        #    critThreads = []

        #    for t in _all_threads:
        #        if t.isCritical():
        #            critThreads.append( t )
        #        else:
        #            nonCritThreads.append( t )

        def __cnt_alive_threads__(_all_threads):
            num_alive_threads = 0
            for t in _all_threads:
                if t.isAlive():
                    num_alive_threads += 1
            return num_alive_threads

        num_alive_threads = __cnt_alive_threads__(_all_threads)

        while num_alive_threads > 0:
            from Ganga.Utility.logging import getLogger
            logger = getLogger('GangaThread')
            # fix for bug #62543 https://savannah.cern.ch/bugs/?62543
            # following 2 lines swapped so that we access no globals between
            # sleep and exit test
            num_alive_threads = __cnt_alive_threads__(_all_threads)
            logger.debug('number of alive threads: %d' % num_alive_threads)
            time.sleep(0.3)
            num_alive_threads = __cnt_alive_threads__(_all_threads)
Пример #6
0
def _ganga_run_exitfuncs():
    """run any registered exit functions

    atexit._exithandlers is traversed based on the priority.
    If no priority was registered for a given function
    than the lowest priority is assumed (LIFO policy)

    We keep the same functionality as in *atexit* bare module but
    we run each exit handler inside a try..catch block to be sure all
    the registered handlers are executed
    """

    from Ganga.GPIDev.Base.Proxy import getName

    #print("Shutting Down Ganga Repositories")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.flush_all()

    from Ganga.Utility.logging import getLogger
    logger = getLogger()

    # Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
    from Ganga.Utility.Config import setConfigOption
    setConfigOption('Configuration', 'DiskIOTimeout', 1)

    try:
        from Ganga.GPI import queues
        queues.lock()
    except Exception as err:
        logger.debug(
            "This should only happen if Ganga filed to initialize correctly")
        logger.debug("Err: %s" % str(err))

    ## Stop the Mon loop from iterating further!
    from Ganga.Core import monitoring_component
    if monitoring_component is not None:
        from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
        getStackTrace()
        monitoring_component.disableMonitoring()
        #monitoring_component.stop()

    ## This will stop the Registries flat but we may still have threads processing data!
    #from Ganga.Core.InternalServices import Coordinator
    #if Coordinator.servicesEnabled:
    #    Coordinator.disableInternalServices( shutdown = True )

    ## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
    from Ganga.GPIDev.Lib.Tasks import stopTasks
    stopTasks()

    # Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
    #from Ganga.Utility.Config import setConfigOption
    #setConfigOption('Configuration', 'DiskIOTimeout', 3)

    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
    _purge_actions_queue()
    stop_and_free_thread_pool()

    try:
        from Ganga.GPI import queues
        queues._purge_all()
    except Exception as err:
        logger.debug(
            "This should only happen if Ganga filed to initialize correctly")
        logger.debug("Err2: %s" % str(err))

    def priority_cmp(f1, f2):
        """
        Sort the exit functions based on priority in reversed order
        """
        # extract the priority number from the function element
        p1 = f1[0][0]
        p2 = f2[0][0]
        # sort in reversed order
        return cmp(p2, p1)

    def add_priority(x):
        """
        add a default priority to the functions not defining one (default priority=sys.maxint)
        return a list containg ((priority,func),*targs,*kargs) elements
        """
        import sys
        func = x[0]
        if isinstance(func, tuple) and len(x[0]) == 2:
            return x
        else:
            new = [(sys.maxsize, func)]
            new.extend(x[1:])
            return new

    atexit._exithandlers = map(add_priority, atexit._exithandlers)
    atexit._exithandlers.sort(priority_cmp)

    logger.info("Stopping running tasks before shutting down Repositories")

    import inspect
    while atexit._exithandlers:

        (priority, func), targs, kargs = atexit._exithandlers.pop()
        try:
            if hasattr(func, 'im_class'):
                for cls in inspect.getmro(func.__self__.__class__):
                    if getName(func) in cls.__dict__:
                        logger.debug(getName(cls) + " : " + getName(func))
                func(*targs, **kargs)
            else:
                logger.debug("noclass : " + getName(func))
                #print("%s" % str(func))
                #print("%s" % str(inspect.getsourcefile(func)))
                #func(*targs, **kargs)
                ## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
                if str(inspect.getsourcefile(func)).find('Ganga') != -1:
                    func(*targs, **kargs)
        except Exception as err:
            s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (
                getName(func), str(err))
            logger.debug(s)
            try:
                import os
                logger.debug("%s" % os.path.join(
                    os.path.dirname(os.path.abspath(inspect.getfile(func)))))
                logger.debug("\n%s" % inspect.getsource(func))
            except Exception as err2:
                logger.debug(
                    "Error getting source code and failure reason: %s" %
                    str(err2))

    logger.info("Shutting Down Ganga Repositories")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.shutdown()

    from Ganga.Core.InternalServices import Coordinator
    Coordinator.servicesEnabled = False

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    from Ganga.Utility.logging import requires_shutdown, final_shutdown
    if requires_shutdown is True:
        final_shutdown()

    from Ganga.Runtime import bootstrap
    if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
        bootstrap.printOpenFiles()
Пример #7
0
    ## This will stop the Registries flat but we may still have threads processing data!
    #from Ganga.Core.InternalServices import Coordinator
    #if Coordinator.servicesEnabled:
    #    Coordinator.disableInternalServices( shutdown = True )

    # Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
    #from Ganga.Utility.Config import setConfigOption
    #setConfigOption('Configuration', 'DiskIOTimeout', 3)

    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
    _purge_actions_queue()
    stop_and_free_thread_pool()

    try:
        from Ganga.GPI import queues
        queues._purge_all()
    except Exception, err:
        logger.debug("This should only happen if Ganga filed to initialize correctly")

    def priority_cmp(f1, f2):
        """
        Sort the exit functions based on priority in reversed order
        """
        # extract the priority number from the function element
        p1 = f1[0][0]
        p2 = f2[0][0]
        # sort in reversed order
        return cmp(p2, p1)

    def add_priority(x):
        """