Example #1
0
def shutdown():
    logger.debug("registry shutdown")
    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)
    try:
        if "prep" in started_registries:
            registry = getRegistry("prep")
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except:
        logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
        logger.error("Trying to shutdown cleanly regardless")
        pass

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
            logger.error("%s" % str(x))
            logger.error("Trying to Shutdown cleanly regardless")
            pass

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers

    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()
Example #2
0
def shutdown():
    # Shutdown method for all repgistries in order
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    logger.info('Registry Shutdown')
    #import traceback
    #traceback.print_stack()

    # Flush all repos before we shut them down
    flush_all()

    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)

    all_registries = getRegistries()

    try:
        if 'prep' in started_registries:
            registry = getRegistry('prep')
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except Exception as err:
        logger.error("Err: %s" % err)
        logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
        logger.error("Trying to shutdown cleanly regardless")

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
            logger.error("%s" % x)
            logger.error("Trying to Shutdown cleanly regardless")


    for registry in all_registries:

        my_reg = [registry]
        if hasattr(registry, 'metadata'):
            if registry.metadata:
                my_reg.append(registry.metadata)

        assigned_attrs = ['location', 'type']
        for this_reg in my_reg:
            for attr in assigned_attrs:
                if hasattr(registry, attr):
                    delattr(registry, attr)

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    removeRegistries()
Example #3
0
def shutdown():
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    logger.info('Registry Shutdown')
    #import traceback
    #traceback.print_stack()
    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)
    try:
        if 'prep' in started_registries:
            registry = getRegistry('prep')
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except Exception as err:
        logger.debug("Err: %s" % str(err))
        logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
        logger.error("Trying to shutdown cleanly regardless")

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
            logger.error("%s" % str(x))
            logger.error("Trying to Shutdown cleanly regardless")

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()
Example #4
0
def shutdown():
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    logger.info('Registry Shutdown')
    #import traceback
    #traceback.print_stack()
    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)
    try:
        if 'prep' in started_registries:
            registry = getRegistry('prep')
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except Exception as err:
        logger.debug("Err: %s" % str(err))
        logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
        logger.error("Trying to shutdown cleanly regardless")

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
            logger.error("%s" % str(x))
            logger.error("Trying to Shutdown cleanly regardless")

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()
Example #5
0
def _ganga_run_exitfuncs():
    """Run all exit functions from plugins and internal services in the correct order

    Go over all plugins and internal services and call the appropriate shutdown functions in the correct order. Because
    we want each shutdown function to be run (e.g. to make sure flushing is done) we put each call into a try..except
    and report to the user before continuing.
    """

    # Set the disk timeout to 1 sec, sacrifice stability for quicker exit
    setConfigOption('Configuration', 'DiskIOTimeout', 1)

    # Stop the monitoring loop from iterating further
    if monitoring_component is not None:
        try:
            getStackTrace()
            if monitoring_component.alive:
                monitoring_component.disableMonitoring()
                monitoring_component.stop()
                monitoring_component.join()
        except Exception as err:
            logger.exception("Exception raised while stopping the monitoring: %s" % err)

    # Stop the tasks system from running
    try:
        stopTasks()
    except Exception as err:
        logger.exception("Exception raised while stopping Tasks: %s" % err)

    # purge the monitoring queues
    try:
        _purge_actions_queue()
        stop_and_free_thread_pool()
    except Exception as err:
        logger.exception("Exception raised while purging monitoring queues: %s" % err)

    # Freeze queues
    try:
        if _global_queues:
            _global_queues.freeze()
    except Exception as err:
        logger.exception("Exception raised during freeze of Global Queues: %s" % err)

    # shutdown the threads in the GangaThreadPool
    try:
        GangaThreadPool.getInstance().shutdown()
    except Exception as err:
        logger.exception("Exception raised during shutdown of GangaThreadPool: %s" % err)

    # Shutdown queues
    try:
        logger.info("Stopping Job processing before shutting down Repositories")
        shutDownQueues()
    except Exception as err:
        logger.exception("Exception raised while purging shutting down queues: %s" % err)

    # shutdown the repositories
    try:
        logger.info("Shutting Down Ganga Repositories")
        Repository_runtime.shutdown()
    except Exception as err:
        logger.exception("Exception raised while shutting down repositories: %s" % err)

    # label services as disabled
    Coordinator.servicesEnabled = False

    # clear the credential store
    try:
        CredentialStore.shutdown()
    except Exception as err:
        logger.exception("Exception raised while clearing the credential store: %s" % err)

    # shutdown SessionLock
    try:
        removeGlobalSessionFileHandlers()
        removeGlobalSessionFiles()
    except Exception as err:
        logger.exception("Exception raised while shutting down SessionLocks: %s" % err)

    # Shutdown stacktracer
    if stacktracer._tracer:
        try:
            stacktracer.trace_stop()
        except Exception as err:
            logger.exception("Exception raised while stopping stack tracer: %s" % err)

    # do final shutdown
    if requires_shutdown is True:
        try:
            final_shutdown()
        except Exception as err:
            logger.exception("Exception raised while doing final shutdown: %s" % err)

    # show any open files after everything's shutdown
    if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
        bootstrap.printOpenFiles()
def _ganga_run_exitfuncs():
    """run any registered exit functions

    atexit._exithandlers is traversed based on the priority.
    If no priority was registered for a given function
    than the lowest priority is assumed (LIFO policy)

    We keep the same functionality as in *atexit* bare module but
    we run each exit handler inside a try..catch block to be sure all
    the registered handlers are executed
    """

    from Ganga.GPIDev.Base.Proxy import getName

    #print("Shutting Down Ganga Repositories")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.flush_all()

    from Ganga.Utility.logging import getLogger
    logger = getLogger()

    # Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
    from Ganga.Utility.Config import setConfigOption
    setConfigOption('Configuration', 'DiskIOTimeout', 1)

    ## Stop the Mon loop from iterating further!
    from Ganga.Core import monitoring_component
    if monitoring_component is not None:
        from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
        getStackTrace()
        monitoring_component.disableMonitoring()

    ## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
    from Ganga.GPIDev.Lib.Tasks import stopTasks
    stopTasks()

    # Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
    #from Ganga.Utility.Config import setConfigOption
    #setConfigOption('Configuration', 'DiskIOTimeout', 3)

    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
    _purge_actions_queue()
    stop_and_free_thread_pool()

    def priority_cmp(f1, f2):
        """
        Sort the exit functions based on priority in reversed order
        """
        # extract the priority number from the function element
        p1 = f1[0][0]
        p2 = f2[0][0]
        # sort in reversed order
        return cmp(p2, p1)

    def add_priority(x):
        """
        add a default priority to the functions not defining one (default priority=sys.maxint)
        return a list containg ((priority,func),*targs,*kargs) elements
        """
        import sys
        func = x[0]
        if isinstance(func, tuple) and len(x[0]) == 2:
            return x
        else:
            new = [(sys.maxsize, func)]
            new.extend(x[1:])
            return new

    atexit._exithandlers = map(add_priority, atexit._exithandlers)
    atexit._exithandlers.sort(priority_cmp)

    logger.info("Stopping Job processing before shutting down Repositories")

    import inspect
    while atexit._exithandlers:

        (priority, func), targs, kargs = atexit._exithandlers.pop()
        try:
            if hasattr(func, 'im_class'):
                for cls in inspect.getmro(func.__self__.__class__):
                    if getName(func) in cls.__dict__:
                        logger.debug(getName(cls) + " : " + getName(func))
                func(*targs, **kargs)
            else:
                logger.debug("noclass : " + getName(func))
                #print("%s" % str(func))
                #print("%s" % str(inspect.getsourcefile(func)))
                #func(*targs, **kargs)
                ## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
                if str(inspect.getsourcefile(func)).find('Ganga') != -1:
                    func(*targs, **kargs)
        except Exception as err:
            s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (getName(func), str(err))
            logger.debug(s)
            try:
                import os
                logger.debug("%s" % os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(func)))) )
                logger.debug("\n%s" % inspect.getsource(func))
            except Exception as err2:
                logger.debug("Error getting source code and failure reason: %s" % str(err2))

    logger.info("Shutting Down Ganga Repositories")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.shutdown()

    from Ganga.Core.InternalServices import Coordinator
    Coordinator.servicesEnabled = False

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    from Ganga.Utility.logging import requires_shutdown, final_shutdown
    if requires_shutdown is True:
        final_shutdown()

    from Ganga.Runtime import bootstrap
    if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
        bootstrap.printOpenFiles()
Example #7
0
def _ganga_run_exitfuncs():
    """Run all exit functions from plugins and internal services in the correct order

    Go over all plugins and internal services and call the appropriate shutdown functions in the correct order. Because
    we want each shutdown function to be run (e.g. to make sure flushing is done) we put each call into a try..except
    and report to the user before continuing.
    """

    # Set the disk timeout to 1 sec, sacrifice stability for quicker exit
    setConfigOption('Configuration', 'DiskIOTimeout', 1)

    # Stop the monitoring loop from iterating further
    if monitoring_component is not None:
        try:
            getStackTrace()
            if monitoring_component.alive:
                monitoring_component.disableMonitoring()
                monitoring_component.stop()
                monitoring_component.join()
        except Exception as err:
            logger.exception(
                "Exception raised while stopping the monitoring: %s" % err)

    # Stop the tasks system from running
    try:
        stopTasks()
    except Exception as err:
        logger.exception("Exception raised while stopping Tasks: %s" % err)

    # purge the monitoring queues
    try:
        _purge_actions_queue()
        stop_and_free_thread_pool()
    except Exception as err:
        logger.exception(
            "Exception raised while purging monitoring queues: %s" % err)

    # Freeze queues
    try:
        if _global_queues:
            _global_queues.freeze()
    except Exception as err:
        logger.exception(
            "Exception raised during freeze of Global Queues: %s" % err)

    # shutdown the threads in the GangaThreadPool
    try:
        GangaThreadPool.getInstance().shutdown()
    except Exception as err:
        logger.exception(
            "Exception raised during shutdown of GangaThreadPool: %s" % err)

    # Shutdown queues
    try:
        logger.info(
            "Stopping Job processing before shutting down Repositories")
        shutDownQueues()
    except Exception as err:
        logger.exception(
            "Exception raised while purging shutting down queues: %s" % err)

    # shutdown the repositories
    try:
        logger.info("Shutting Down Ganga Repositories")
        Repository_runtime.shutdown()
    except Exception as err:
        logger.exception(
            "Exception raised while shutting down repositories: %s" % err)

    # label services as disabled
    Coordinator.servicesEnabled = False

    # shutdown SessionLock
    try:
        removeGlobalSessionFileHandlers()
        removeGlobalSessionFiles()
    except Exception as err:
        logger.exception(
            "Exception raised while shutting down SessionLocks: %s" % err)

    # Shutdown stacktracer
    if stacktracer._tracer:
        try:
            stacktracer.trace_stop()
        except Exception as err:
            logger.exception(
                "Exception raised while stopping stack tracer: %s" % err)

    # do final shutdown
    if requires_shutdown is True:
        try:
            final_shutdown()
        except Exception as err:
            logger.exception(
                "Exception raised while doing final shutdown: %s" % err)

    # show any open files after everything's shutdown
    if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
        bootstrap.printOpenFiles()
Example #8
0
def _ganga_run_exitfuncs():
    """run any registered exit functions

    atexit._exithandlers is traversed based on the priority.
    If no priority was registered for a given function
    than the lowest priority is assumed (LIFO policy)

    We keep the same functionality as in *atexit* bare module but
    we run each exit handler inside a try..catch block to be sure all
    the registered handlers are executed
    """

    from Ganga.GPIDev.Base.Proxy import getName

    #print("Shutting Down Ganga Repositories")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.flush_all()

    from Ganga.Utility.logging import getLogger
    logger = getLogger()

    # Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
    from Ganga.Utility.Config import setConfigOption
    setConfigOption('Configuration', 'DiskIOTimeout', 1)

    try:
        from Ganga.GPI import queues
        queues.lock()
    except Exception as err:
        logger.debug(
            "This should only happen if Ganga filed to initialize correctly")
        logger.debug("Err: %s" % str(err))

    ## Stop the Mon loop from iterating further!
    from Ganga.Core import monitoring_component
    if monitoring_component is not None:
        from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
        getStackTrace()
        monitoring_component.disableMonitoring()
        #monitoring_component.stop()

    ## This will stop the Registries flat but we may still have threads processing data!
    #from Ganga.Core.InternalServices import Coordinator
    #if Coordinator.servicesEnabled:
    #    Coordinator.disableInternalServices( shutdown = True )

    ## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
    from Ganga.GPIDev.Lib.Tasks import stopTasks
    stopTasks()

    # Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
    #from Ganga.Utility.Config import setConfigOption
    #setConfigOption('Configuration', 'DiskIOTimeout', 3)

    from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
    _purge_actions_queue()
    stop_and_free_thread_pool()

    try:
        from Ganga.GPI import queues
        queues._purge_all()
    except Exception as err:
        logger.debug(
            "This should only happen if Ganga filed to initialize correctly")
        logger.debug("Err2: %s" % str(err))

    def priority_cmp(f1, f2):
        """
        Sort the exit functions based on priority in reversed order
        """
        # extract the priority number from the function element
        p1 = f1[0][0]
        p2 = f2[0][0]
        # sort in reversed order
        return cmp(p2, p1)

    def add_priority(x):
        """
        add a default priority to the functions not defining one (default priority=sys.maxint)
        return a list containg ((priority,func),*targs,*kargs) elements
        """
        import sys
        func = x[0]
        if isinstance(func, tuple) and len(x[0]) == 2:
            return x
        else:
            new = [(sys.maxsize, func)]
            new.extend(x[1:])
            return new

    atexit._exithandlers = map(add_priority, atexit._exithandlers)
    atexit._exithandlers.sort(priority_cmp)

    logger.info("Stopping running tasks before shutting down Repositories")

    import inspect
    while atexit._exithandlers:

        (priority, func), targs, kargs = atexit._exithandlers.pop()
        try:
            if hasattr(func, 'im_class'):
                for cls in inspect.getmro(func.__self__.__class__):
                    if getName(func) in cls.__dict__:
                        logger.debug(getName(cls) + " : " + getName(func))
                func(*targs, **kargs)
            else:
                logger.debug("noclass : " + getName(func))
                #print("%s" % str(func))
                #print("%s" % str(inspect.getsourcefile(func)))
                #func(*targs, **kargs)
                ## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
                if str(inspect.getsourcefile(func)).find('Ganga') != -1:
                    func(*targs, **kargs)
        except Exception as err:
            s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (
                getName(func), str(err))
            logger.debug(s)
            try:
                import os
                logger.debug("%s" % os.path.join(
                    os.path.dirname(os.path.abspath(inspect.getfile(func)))))
                logger.debug("\n%s" % inspect.getsource(func))
            except Exception as err2:
                logger.debug(
                    "Error getting source code and failure reason: %s" %
                    str(err2))

    logger.info("Shutting Down Ganga Repositories")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.shutdown()

    from Ganga.Core.InternalServices import Coordinator
    Coordinator.servicesEnabled = False

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    from Ganga.Utility.logging import requires_shutdown, final_shutdown
    if requires_shutdown is True:
        final_shutdown()

    from Ganga.Runtime import bootstrap
    if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
        bootstrap.printOpenFiles()
Example #9
0
            logger.warning(s)

    logger.debug("Shutting Down Repository_runtime")
    from Ganga.Runtime import Repository_runtime
    Repository_runtime.shutdown()

    import Ganga.Utility.logging
    if Ganga.Utility.logging.requires_shutdown is True:
        Ganga.Utility.logging.shutdown()

    from Ganga.Core.InternalServices import Coordinator
    Coordinator.servicesEnabled = False

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    from Ganga.Runtime import bootstrap
    if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
        bootstrap.printOpenFiles()


def install():
    """
    Install a new shutdown manager, by overriding methods from atexit module
    """
    # override the atexit exit function
    atexit._run_exitfuncs = _ganga_run_exitfuncs
    #del atexit

    # override the default exit function
Example #10
0
def shutdown():
    # Shutdown method for all repgistries in order
    from Ganga.Utility.logging import getLogger
    logger = getLogger()
    logger.info('Registry Shutdown')
    #import traceback
    #traceback.print_stack()

    # Flush all repos before we shut them down
    flush_all()

    # shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
    # method to perform actions on the box and/or job registries.
    logger.debug(started_registries)

    all_registries = getRegistries()

    try:
        if 'prep' in started_registries:
            registry = getRegistry('prep')
            registry.shutdown()
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(registry.name)
    except Exception as err:
        logger.error("Err: %s" % err)
        logger.error(
            "Failed to Shutdown prep Repository!!! please check for stale lock files"
        )
        logger.error("Trying to shutdown cleanly regardless")

    for registry in getRegistries():
        thisName = registry.name
        try:
            if not thisName in started_registries:
                continue
            # in case this is called repeatedly, only call shutdown once
            started_registries.remove(thisName)
            registry.shutdown()  # flush and release locks
        except Exception as x:
            logger.error(
                "Failed to Shutdown Repository: %s !!! please check for stale lock files"
                % thisName)
            logger.error("%s" % x)
            logger.error("Trying to Shutdown cleanly regardless")

    for registry in all_registries:

        my_reg = [registry]
        if hasattr(registry, 'metadata'):
            if registry.metadata:
                my_reg.append(registry.metadata)

        assigned_attrs = ['location', 'type']
        for this_reg in my_reg:
            for attr in assigned_attrs:
                if hasattr(registry, attr):
                    delattr(registry, attr)

    from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
    removeGlobalSessionFileHandlers()
    removeGlobalSessionFiles()

    removeRegistries()