def enableInternalServices():
"""
activates the internal services previously disabled due to expired credentials
"""
global servicesEnabled
if servicesEnabled:
log.error("Cannot (re)enable services, they're already running")
from Ganga.Core.exceptions import GangaException
raise GangaException("Cannot (re)enable services")
# startup the registries
from Ganga.Runtime import Repository_runtime
Repository_runtime.bootstrap()
# make sure all required credentials are valid
missing_cred = getMissingCredentials()
if missing_cred:
log.error("The following credentials are still required: %s."
"Make sure you renew them before reactivating this session" %
','.join(missing_cred))
return
log.debug("Enabling the internal services")
# re-enable the monitoring loop as it's been explicityly requested here
enableMonitoringService()
servicesEnabled = True
log.info('Internal services reactivated successfuly')
def isCredentialRequired(credObj):
"""
The logic to decide if a given invalid credential
should trigger the deactivation of Ganga internal services.
"""
from Ganga.Runtime import Workspace_runtime
from Ganga.Runtime import Repository_runtime
if getName(credObj) == "AfsToken":
return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()
if getName(credObj) == "GridProxy":
if Repository_runtime.requiresGridProxy() or Workspace_runtime.requiresGridProxy():
return True
from Ganga.GPI import jobs, typename
from Ganga.GPIDev.Base.Proxy import stripProxy
for j in jobs:
ji = stripProxy(j)
if ji.status in ["submitted", "running", "completing"]:
if getName(ji.backend) == "LCG":
return True
return False
log.warning("Unknown credential object : %s" % credObj)
def enableInternalServices():
"""
activates the internal services previously disabled due to expired credentials
"""
global servicesEnabled
if servicesEnabled:
log.error("Cannot (re)enable services, they're already running")
from Ganga.Core.exceptions import GangaException
raise GangaException("Cannot (re)enable services")
# startup the registries
from Ganga.Runtime import Repository_runtime
Repository_runtime.bootstrap()
# make sure all required credentials are valid
missing_cred = getMissingCredentials()
if missing_cred:
log.error("The following credentials are still required: %s."
"Make sure you renew them before reactivating this session" % ','.join(missing_cred))
return
log.debug("Enabling the internal services")
# re-enable the monitoring loop as it's been explicityly requested here
enableMonitoringService()
servicesEnabled = True
log.info('Internal services reactivated successfuly')
def isCredentialRequired(credObj):
"""
The logic to decide if a given invalid credential
should trigger the deactivation of Ganga internal services.
"""
from Ganga.Runtime import Workspace_runtime
from Ganga.Runtime import Repository_runtime
if getName(credObj) == 'AfsToken':
return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()
if getName(credObj) == 'GridProxy':
if Repository_runtime.requiresGridProxy() or Workspace_runtime.requiresGridProxy():
return True
from Ganga.GPI import jobs, typename
from Ganga.GPIDev.Base.Proxy import stripProxy
for j in jobs:
ji = stripProxy(j)
if ji.status in ['submitted', 'running', 'completing']:
if ji.getNodeIndexCache() is not None and 'display:backend' in ji.getNodeIndexCache().keys():
if ji.getNodeIndexCache()['display:backend'] == 'LCG':
return True
else:
if getName(ji.backend) == 'LCG':
return True
return False
log.warning("Unknown credential object : %s" % credObj)
def __init__(self, repo=None, what=''):
"""
This is a fatal repo error
Args:
repo (GangaRepository): The repository the error happened in
what (str): The original exception/error/description
"""
super(RepositoryError, self).__init__(self, what)
self.what = what
self.repository = repo
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.error("A severe error occurred in the Repository '%s': %s" %
(repo.registry.name, what))
logger.error(
'If you believe the problem has been solved, type "reactivate()" to re-enable '
)
try:
from Ganga.Core.InternalServices.Coordinator import disableInternalServices
disableInternalServices()
from Ganga.Core.GangaThread.WorkerThreads import shutDownQueues
shutDownQueues()
logger.error("Shutting Down Repository_runtime")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
except:
logger.error(
"Unable to disable Internal services, they may have already been disabled!"
)
def enableInternalServices():
"""
activates the internal services previously disabled due to expired credentials
"""
global servicesEnabled
if servicesEnabled:
log.error("Cannot (re)enable services, they're already running")
from Ganga.Core.exceptions import GangaException
raise GangaException("Cannot (re)enable services")
# startup the registries
from Ganga.Runtime import Repository_runtime
Repository_runtime.bootstrap()
# make sure all required credentials are valid
invalid_afs = [
afsToken
for afsToken in credential_store.get_all_matching_type(AfsToken)
if not afsToken.is_valid()
]
if invalid_afs:
log.error(
'No valid AFS token was found. Please re-authorise before reactivating this session.'
)
return
log.debug("Enabling the internal services")
# re-enable the monitoring loop as it's been explicityly requested here
enableMonitoringService()
servicesEnabled = True
log.info('Internal services reactivated successfuly')
def enableInternalServices():
"""
activates the internal services previously disabled due to expired credentials
"""
global servicesEnabled
if servicesEnabled:
log.error("Cannot (re)enable services, they're already running")
from Ganga.Core.exceptions import GangaException
raise GangaException("Cannot (re)enable services")
# startup the registries
from Ganga.Runtime import Repository_runtime
Repository_runtime.bootstrap()
# make sure all required credentials are valid
invalid_afs = [afsToken for afsToken in credential_store.get_all_matching_type(AfsToken) if not afsToken.is_valid()]
if invalid_afs:
log.error('No valid AFS token was found. Please re-authorise before reactivating this session.')
return
log.debug("Enabling the internal services")
# re-enable the monitoring loop as it's been explicityly requested here
enableMonitoringService()
servicesEnabled = True
log.info('Internal services reactivated successfuly')
def __init__(self, repo=None, what=''):
GangaException.__init__(self, what)
self.what = what
self.repository = repo
logger.error("A severe error occurred in the Repository '%s': %s" % (repo.registry.name, what))
logger.error('If you believe the problem has been solved, type "reactivate()" to re-enable ')
try:
from Ganga.Core.InternalServices.Coordinator import disableInternalServices
disableInternalServices()
logger.error("Shutting Down Repository_runtime")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
except:
logger.error("Unable to disable Internal services, they may have already been disabled!")
def isCredentialRequired(credObj):
"""
The logic to decide if a given invalid credential
should trigger the deactivation of Ganga internal services.
"""
from Ganga.Runtime import Workspace_runtime
from Ganga.Runtime import Repository_runtime
if getName(credObj) == 'AfsToken':
return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()
if getName(credObj) == 'GridProxy':
from Ganga.Core.GangaRepository import getRegistryProxy
from Ganga.Runtime.GPIFunctions import typename
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.GPIDev.Lib.Job.Job import lazyLoadJobBackend, lazyLoadJobStatus
for j in getRegistryProxy('jobs'):
ji = stripProxy(j)
this_status = lazyLoadJobStatus(ji)
if this_status in ['submitted', 'running', 'completing']:
this_backend = lazyLoadJobBackend(ji)
if getName(this_backend) == 'LCG':
return True
return False
log.warning("Unknown credential object : %s" % credObj)
def isCredentialRequired(credObj):
"""
The logic to decide if a given invalid credential
should trigger the deactivation of Ganga internal services.
"""
from Ganga.Runtime import Workspace_runtime
from Ganga.Runtime import Repository_runtime
if getName(credObj) == 'AfsToken':
return Workspace_runtime.requiresAfsToken(
) or Repository_runtime.requiresAfsToken()
if getName(credObj) == 'GridProxy':
from Ganga.Core.GangaRepository import getRegistryProxy
from Ganga.Runtime.GPIFunctions import typename
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.GPIDev.Lib.Job.Job import lazyLoadJobBackend, lazyLoadJobStatus
for j in getRegistryProxy('jobs'):
ji = stripProxy(j)
this_status = lazyLoadJobStatus(ji)
if this_status in ['submitted', 'running', 'completing']:
this_backend = lazyLoadJobBackend(ji)
if getName(this_backend) == 'LCG':
return True
return False
log.warning("Unknown credential object : %s" % credObj)
def __init__(self, repo=None, what=''):
GangaException.__init__(self, what)
self.what = what
self.repository = repo
logger.error("A severe error occurred in the Repository '%s': %s" %
(repo.registry.name, what))
logger.error(
'If you believe the problem has been solved, type "reactivate()" to re-enable '
)
try:
from Ganga.Core.InternalServices.Coordinator import disableInternalServices
disableInternalServices()
logger.error("Shutting Down Repository_runtime")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
except:
logger.error(
"Unable to disable Internal services, they may have already been disabled!"
)
def isCredentialRequired (credObj):
"""
The logic to decide if a given invalid credential
should trigger the deactivation of Ganga internal services.
"""
from Ganga.Runtime import Workspace_runtime
from Ganga.Runtime import Repository_runtime
if credObj.__class__.__name__ == 'AfsToken':
return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()
if credObj.__class__.__name__ == 'GridProxy':
if Repository_runtime.requiresGridProxy() or Workspace_runtime.requiresGridProxy():
return True
from Ganga.GPI import jobs,typename
return bool([j for j in jobs if typename(j.backend)=='LCG' and j.status in ['submitted','running','completing']])
log.warning("Unknown credential object : %s" % credObj)
def disableInternalServices():
"""
Deactivates all the internal services :
* monitoring loop
* registry/repository and workspace (or GPI entierly)
Currently this method is called whenever:
* one of the managed credentials (AFS token or Grid Proxy) is detected as beeing *invalid* by the monitoring component
* the user is running out of space
"""
global servicesEnabled
log.debug("Disabling the internal services")
#disable the mon loop
from Ganga.Core import monitoring_component
monitoring_component.disableMonitoring()
#flush the registries
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
#this will disable any interactions with the registries (implicitly with the GPI)
servicesEnabled = False
def enableInternalServices():
"""
activates the internal services previously disabled due to expired credentials
"""
global servicesEnabled
#make sure all required credentials are valid
missing_cred = getMissingCredentials()
if missing_cred:
log.error("The following credentials are still required: %s."
"Make sure you renew them before reactivating this session" % ','.join(missing_cred))
return
#startup the registries
from Ganga.Runtime import Repository_runtime
Repository_runtime.bootstrap()
log.debug("Enabling the internal services")
# reenable the monitoring loop if *autostart* is set
from Ganga.Core import monitoring_component
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import config
if config['autostart']:
monitoring_component.enableMonitoring()
servicesEnabled = True
log.info('Internal services reactivated successfuly')
def removeRegistries():
## Remove lingering Objects from the GPI and fully cleanup after the startup
## First start with repositories
import Ganga.GPI
from Ganga.Runtime import Repository_runtime
for name in Repository_runtime.bootstrap_reg_names():
if hasattr(Ganga.GPI, name):
delattr(Ganga.GPI, name)
if hasattr(Ganga.GPI, 'jobtree'):
## Now remove the JobTree
delattr(Ganga.GPI, 'jobtree')
if hasattr(Ganga.GPI, 'shareref'):
## Now remove the sharedir
delattr(Ganga.GPI, 'shareref')
def removeRegistries():
## Remove lingering Objects from the GPI and fully cleanup after the startup
## First start with repositories
import Ganga.GPI
from Ganga.Runtime import Repository_runtime
to_remove = Repository_runtime.bootstrap_reg_names()
to_remove.append('jobtree')
to_remove.append('shareref')
global _runtime_interface
for name in to_remove:
if hasattr(_runtime_interface, name):
delattr(_runtime_interface, name)
_runtime_interface = None
def removeRegistries():
## Remove lingering Objects from the GPI and fully cleanup after the startup
## First start with repositories
import Ganga.GPI
from Ganga.Runtime import Repository_runtime
to_remove = Repository_runtime.bootstrap_reg_names()
to_remove.append('jobtree')
to_remove.append('shareref')
global _runtime_interface
for name in to_remove:
if hasattr(_runtime_interface, name):
delattr(_runtime_interface, name)
_runtime_interface = None
def isAvailable( self ):
if self.cell:
available = True
else:
available = False
if not available:
# gangadir = fullpath( getConfig( "Configuration" )[ "gangadir" ] )
# if ( 0 == gangadir.find( "/afs" ) ):
# available = True
available = Repository_runtime.requiresAfsToken()
if available:
infoCommand = self.command.info.split()[ 0 ]
available = False
try:
pathList = os.environ[ "PATH" ].split( os.pathsep )
except KeyError:
pathList = []
for searchDir in pathList:
try:
fileList = os.listdir( searchDir )
except OSError:
fileList = []
if infoCommand in fileList:
available = True
break
if available:
logger.debug( "Command '%s' found in directory '%s'" % \
( infoCommand, searchDir ) )
else:
logger.debug( "Unable to find command '%s'" % infoCommand )
if available:
timeleft = self.timeleft()
if not timeleft:
available = False
return available
def isAvailable(self):
if self.cell:
available = True
else:
available = False
if not available:
available = Repository_runtime.requiresAfsToken()
if available:
infoCommand = self.command.info.split()[0]
available = False
try:
pathList = os.environ["PATH"].split(os.pathsep)
except KeyError:
pathList = []
for searchDir in pathList:
try:
fileList = os.listdir(searchDir)
except OSError:
fileList = []
if infoCommand in fileList:
available = True
break
if available:
logger.debug("Command '%s' found in directory '%s'" %
(infoCommand, searchDir))
else:
logger.debug("Unable to find command '%s'" % infoCommand)
if available:
timeleft = self.timeleft()
if not timeleft:
available = False
return available
exportToPublicInterface('load', load, 'Functions')
exportToPublicInterface('export', export, 'Functions')
exportToPublicInterface('typename', typename, 'Functions')
exportToPublicInterface('categoryname', categoryname, 'Functions')
exportToPublicInterface('plugins', plugins, 'Functions')
exportToPublicInterface('convert_merger_to_postprocessor',
convert_merger_to_postprocessor, 'Functions')
exportToPublicInterface('config', Ganga.GPIDev.Lib.Config.config, 'Objects',
'access to Ganga configuration')
exportToPublicInterface('ConfigError', Ganga.GPIDev.Lib.Config.ConfigError,
'Exceptions')
exportToPublicInterface('report', report, 'Functions')
# ------------------------------------------------------------------------------------
# bootstrap the repositories and connect to them
for n, k, d in Repository_runtime.bootstrap():
# make all repository proxies visible in GPI
exportToPublicInterface(n, k, 'Objects', d)
# ------------------------------------------------------------------------------------
# JobTree
jobtree = GPIProxyObjectFactory(getRegistry("jobs").getJobTree())
exportToPublicInterface('jobtree', jobtree, 'Objects',
'Logical tree view of the jobs')
exportToPublicInterface('TreeError', TreeError, 'Exceptions')
# ------------------------------------------------------------------------------------
# ShareRef
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
exportToPublicInterface(
'shareref', shareref, 'Objects',
def _ganga_run_exitfuncs():
"""run any registered exit functions
atexit._exithandlers is traversed based on the priority.
If no priority was registered for a given function
than the lowest priority is assumed (LIFO policy)
We keep the same functionality as in *atexit* bare module but
we run each exit handler inside a try..catch block to be sure all
the registered handlers are executed
"""
from Ganga.GPIDev.Base.Proxy import getName
#print("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.flush_all()
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'DiskIOTimeout', 1)
try:
from Ganga.GPI import queues
queues.lock()
except Exception as err:
logger.debug(
"This should only happen if Ganga filed to initialize correctly")
logger.debug("Err: %s" % str(err))
## Stop the Mon loop from iterating further!
from Ganga.Core import monitoring_component
if monitoring_component is not None:
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
getStackTrace()
monitoring_component.disableMonitoring()
#monitoring_component.stop()
## This will stop the Registries flat but we may still have threads processing data!
#from Ganga.Core.InternalServices import Coordinator
#if Coordinator.servicesEnabled:
# Coordinator.disableInternalServices( shutdown = True )
## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
from Ganga.GPIDev.Lib.Tasks import stopTasks
stopTasks()
# Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
#from Ganga.Utility.Config import setConfigOption
#setConfigOption('Configuration', 'DiskIOTimeout', 3)
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
try:
from Ganga.GPI import queues
queues._purge_all()
except Exception as err:
logger.debug(
"This should only happen if Ganga filed to initialize correctly")
logger.debug("Err2: %s" % str(err))
def priority_cmp(f1, f2):
"""
Sort the exit functions based on priority in reversed order
"""
# extract the priority number from the function element
p1 = f1[0][0]
p2 = f2[0][0]
# sort in reversed order
return cmp(p2, p1)
def add_priority(x):
"""
add a default priority to the functions not defining one (default priority=sys.maxint)
return a list containg ((priority,func),*targs,*kargs) elements
"""
import sys
func = x[0]
if isinstance(func, tuple) and len(x[0]) == 2:
return x
else:
new = [(sys.maxsize, func)]
new.extend(x[1:])
return new
atexit._exithandlers = map(add_priority, atexit._exithandlers)
atexit._exithandlers.sort(priority_cmp)
logger.info("Stopping running tasks before shutting down Repositories")
import inspect
while atexit._exithandlers:
(priority, func), targs, kargs = atexit._exithandlers.pop()
try:
if hasattr(func, 'im_class'):
for cls in inspect.getmro(func.__self__.__class__):
if getName(func) in cls.__dict__:
logger.debug(getName(cls) + " : " + getName(func))
func(*targs, **kargs)
else:
logger.debug("noclass : " + getName(func))
#print("%s" % str(func))
#print("%s" % str(inspect.getsourcefile(func)))
#func(*targs, **kargs)
## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
if str(inspect.getsourcefile(func)).find('Ganga') != -1:
func(*targs, **kargs)
except Exception as err:
s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (
getName(func), str(err))
logger.debug(s)
try:
import os
logger.debug("%s" % os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(func)))))
logger.debug("\n%s" % inspect.getsource(func))
except Exception as err2:
logger.debug(
"Error getting source code and failure reason: %s" %
str(err2))
logger.info("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
from Ganga.Core.InternalServices import Coordinator
Coordinator.servicesEnabled = False
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
from Ganga.Utility.logging import requires_shutdown, final_shutdown
if requires_shutdown is True:
final_shutdown()
from Ganga.Runtime import bootstrap
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
def _ganga_run_exitfuncs():
"""Run all exit functions from plugins and internal services in the correct order
Go over all plugins and internal services and call the appropriate shutdown functions in the correct order. Because
we want each shutdown function to be run (e.g. to make sure flushing is done) we put each call into a try..except
and report to the user before continuing.
"""
# Set the disk timeout to 1 sec, sacrifice stability for quicker exit
setConfigOption('Configuration', 'DiskIOTimeout', 1)
# Stop the monitoring loop from iterating further
if monitoring_component is not None:
try:
getStackTrace()
if monitoring_component.alive:
monitoring_component.disableMonitoring()
monitoring_component.stop()
monitoring_component.join()
except Exception as err:
logger.exception(
"Exception raised while stopping the monitoring: %s" % err)
# Stop the tasks system from running
try:
stopTasks()
except Exception as err:
logger.exception("Exception raised while stopping Tasks: %s" % err)
# purge the monitoring queues
try:
_purge_actions_queue()
stop_and_free_thread_pool()
except Exception as err:
logger.exception(
"Exception raised while purging monitoring queues: %s" % err)
# Freeze queues
try:
if _global_queues:
_global_queues.freeze()
except Exception as err:
logger.exception(
"Exception raised during freeze of Global Queues: %s" % err)
# shutdown the threads in the GangaThreadPool
try:
GangaThreadPool.getInstance().shutdown()
except Exception as err:
logger.exception(
"Exception raised during shutdown of GangaThreadPool: %s" % err)
# Shutdown queues
try:
logger.info(
"Stopping Job processing before shutting down Repositories")
shutDownQueues()
except Exception as err:
logger.exception(
"Exception raised while purging shutting down queues: %s" % err)
# shutdown the repositories
try:
logger.info("Shutting Down Ganga Repositories")
Repository_runtime.shutdown()
except Exception as err:
logger.exception(
"Exception raised while shutting down repositories: %s" % err)
# label services as disabled
Coordinator.servicesEnabled = False
# shutdown SessionLock
try:
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
except Exception as err:
logger.exception(
"Exception raised while shutting down SessionLocks: %s" % err)
# Shutdown stacktracer
if stacktracer._tracer:
try:
stacktracer.trace_stop()
except Exception as err:
logger.exception(
"Exception raised while stopping stack tracer: %s" % err)
# do final shutdown
if requires_shutdown is True:
try:
final_shutdown()
except Exception as err:
logger.exception(
"Exception raised while doing final shutdown: %s" % err)
# show any open files after everything's shutdown
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
def _ganga_run_exitfuncs():
"""run any registered exit functions
atexit._exithandlers is traversed based on the priority.
If no priority was registered for a given function
than the lowest priority is assumed (LIFO policy)
We keep the same functionality as in *atexit* bare module but
we run each exit handler inside a try..catch block to be sure all
the registered handlers are executed
"""
from Ganga.GPIDev.Base.Proxy import getName
#print("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.flush_all()
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'DiskIOTimeout', 1)
## Stop the Mon loop from iterating further!
from Ganga.Core import monitoring_component
if monitoring_component is not None:
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
getStackTrace()
monitoring_component.disableMonitoring()
## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
from Ganga.GPIDev.Lib.Tasks import stopTasks
stopTasks()
# Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
#from Ganga.Utility.Config import setConfigOption
#setConfigOption('Configuration', 'DiskIOTimeout', 3)
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
def priority_cmp(f1, f2):
"""
Sort the exit functions based on priority in reversed order
"""
# extract the priority number from the function element
p1 = f1[0][0]
p2 = f2[0][0]
# sort in reversed order
return cmp(p2, p1)
def add_priority(x):
"""
add a default priority to the functions not defining one (default priority=sys.maxint)
return a list containg ((priority,func),*targs,*kargs) elements
"""
import sys
func = x[0]
if isinstance(func, tuple) and len(x[0]) == 2:
return x
else:
new = [(sys.maxsize, func)]
new.extend(x[1:])
return new
atexit._exithandlers = map(add_priority, atexit._exithandlers)
atexit._exithandlers.sort(priority_cmp)
logger.info("Stopping Job processing before shutting down Repositories")
import inspect
while atexit._exithandlers:
(priority, func), targs, kargs = atexit._exithandlers.pop()
try:
if hasattr(func, 'im_class'):
for cls in inspect.getmro(func.__self__.__class__):
if getName(func) in cls.__dict__:
logger.debug(getName(cls) + " : " + getName(func))
func(*targs, **kargs)
else:
logger.debug("noclass : " + getName(func))
#print("%s" % str(func))
#print("%s" % str(inspect.getsourcefile(func)))
#func(*targs, **kargs)
## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
if str(inspect.getsourcefile(func)).find('Ganga') != -1:
func(*targs, **kargs)
except Exception as err:
s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (getName(func), str(err))
logger.debug(s)
try:
import os
logger.debug("%s" % os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(func)))) )
logger.debug("\n%s" % inspect.getsource(func))
except Exception as err2:
logger.debug("Error getting source code and failure reason: %s" % str(err2))
logger.info("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
from Ganga.Core.InternalServices import Coordinator
Coordinator.servicesEnabled = False
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
from Ganga.Utility.logging import requires_shutdown, final_shutdown
if requires_shutdown is True:
final_shutdown()
from Ganga.Runtime import bootstrap
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
(priority, func), targs, kargs = atexit._exithandlers.pop()
try:
if hasattr(func, 'im_class'):
for cls in inspect.getmro(func.__self__.__class__):
if func.__name__ in cls.__dict__:
logger.debug(cls.__name__ + " : " + func.__name__)
else:
logger.debug("noclass : " + func.__name__)
func(*targs, **kargs)
except Exception as x:
s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (func.__name__, str(x))
logger.warning(s)
logger.debug("Shutting Down Repository_runtime")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
import Ganga.Utility.logging
if Ganga.Utility.logging.requires_shutdown is True:
Ganga.Utility.logging.shutdown()
from Ganga.Core.InternalServices import Coordinator
Coordinator.servicesEnabled = False
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
from Ganga.Runtime import bootstrap
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
def _ganga_run_exitfuncs():
"""Run all exit functions from plugins and internal services in the correct order
Go over all plugins and internal services and call the appropriate shutdown functions in the correct order. Because
we want each shutdown function to be run (e.g. to make sure flushing is done) we put each call into a try..except
and report to the user before continuing.
"""
# Set the disk timeout to 1 sec, sacrifice stability for quicker exit
setConfigOption('Configuration', 'DiskIOTimeout', 1)
# Stop the monitoring loop from iterating further
if monitoring_component is not None:
try:
getStackTrace()
if monitoring_component.alive:
monitoring_component.disableMonitoring()
monitoring_component.stop()
monitoring_component.join()
except Exception as err:
logger.exception("Exception raised while stopping the monitoring: %s" % err)
# Stop the tasks system from running
try:
stopTasks()
except Exception as err:
logger.exception("Exception raised while stopping Tasks: %s" % err)
# purge the monitoring queues
try:
_purge_actions_queue()
stop_and_free_thread_pool()
except Exception as err:
logger.exception("Exception raised while purging monitoring queues: %s" % err)
# Freeze queues
try:
if _global_queues:
_global_queues.freeze()
except Exception as err:
logger.exception("Exception raised during freeze of Global Queues: %s" % err)
# shutdown the threads in the GangaThreadPool
try:
GangaThreadPool.getInstance().shutdown()
except Exception as err:
logger.exception("Exception raised during shutdown of GangaThreadPool: %s" % err)
# Shutdown queues
try:
logger.info("Stopping Job processing before shutting down Repositories")
shutDownQueues()
except Exception as err:
logger.exception("Exception raised while purging shutting down queues: %s" % err)
# shutdown the repositories
try:
logger.info("Shutting Down Ganga Repositories")
Repository_runtime.shutdown()
except Exception as err:
logger.exception("Exception raised while shutting down repositories: %s" % err)
# label services as disabled
Coordinator.servicesEnabled = False
# clear the credential store
try:
CredentialStore.shutdown()
except Exception as err:
logger.exception("Exception raised while clearing the credential store: %s" % err)
# shutdown SessionLock
try:
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
except Exception as err:
logger.exception("Exception raised while shutting down SessionLocks: %s" % err)
# Shutdown stacktracer
if stacktracer._tracer:
try:
stacktracer.trace_stop()
except Exception as err:
logger.exception("Exception raised while stopping stack tracer: %s" % err)
# do final shutdown
if requires_shutdown is True:
try:
final_shutdown()
except Exception as err:
logger.exception("Exception raised while doing final shutdown: %s" % err)
# show any open files after everything's shutdown
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
exportToPublicInterface('load', load, 'Functions')
exportToPublicInterface('export', export, 'Functions')
exportToPublicInterface('typename', typename, 'Functions')
exportToPublicInterface('categoryname', categoryname, 'Functions')
exportToPublicInterface('plugins', plugins, 'Functions')
exportToPublicInterface('convert_merger_to_postprocessor',
convert_merger_to_postprocessor, 'Functions')
exportToPublicInterface('config', Ganga.GPIDev.Lib.Config.config,
'Objects', 'access to Ganga configuration')
exportToPublicInterface('ConfigError', Ganga.GPIDev.Lib.Config.ConfigError,
'Exceptions')
exportToPublicInterface('report', report, 'Functions')
# ------------------------------------------------------------------------------------
# bootstrap the repositories and connect to them
for n, k, d in Repository_runtime.bootstrap():
# make all repository proxies visible in GPI
exportToPublicInterface(n, k, 'Objects', d)
# ------------------------------------------------------------------------------------
# JobTree
jobtree = GPIProxyObjectFactory(getRegistry("jobs").getJobTree())
exportToPublicInterface(
'jobtree', jobtree, 'Objects', 'Logical tree view of the jobs')
exportToPublicInterface('TreeError', TreeError, 'Exceptions')
# ------------------------------------------------------------------------------------
# ShareRef
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
exportToPublicInterface('shareref', shareref, 'Objects',
'Mechanism for tracking use of shared directory resources')
def __init__(self, job_info):
IMonitoringService.__init__(self, job_info)
if isinstance(job_info, DictionaryType):
# we are on the worker node. We just need
# to get values from the dictionary
try:
self.gangaJobId = job_info['gangaJobId']
self.gangaTaskId = job_info['gangaTaskId']
self.gridBackend = job_info['gridBackend']
self.gridCertificate = job_info['gridCertificate']
self.VO = job_info['VO']
self._complete = True
except KeyError as msg:
# too bad, we will not monitor the job
return
# on the WN, we get the job ID from envar
self.gridJobId = safe_getenv('EDG_WL_JOBID')
if self.gridJobId == 'unknown':
self.gridJobId = safe_getenv('GLITE_WL_JOBID')
if self.gridJobId == 'unknown':
self._complete = False
else:
# we are in the client session. job_info is a Job()
from Ganga.Utility.logging import getLogger
self._logger = getLogger()
job = job_info
self.gridBackend = getattr(job, 'backend')._name
if self.gridBackend not in ['LCG']:
self._logger.debug('not sending monitoring because not in LCG')
return
self._logger.debug(job.backend)
self._logger.debug(job.backend.id)
self.gridJobId = job.backend.id
# we compute the "jobID" and "taskID"
# (which is the gangaJobId followed by the user@repository
# the repository unique ID:
from Ganga.Utility.Config import getConfig, ConfigError
config = getConfig('Configuration')
rep_type = config['repositorytype']
rep_login = config['user']
if 'Local' in rep_type:
from Ganga.Runtime import Repository_runtime
rep_dir = Repository_runtime.getLocalRoot()
sys_config = getConfig('System')
rep_hostname = sys_config['GANGA_HOSTNAME']
rep_location = rep_hostname + ':' + rep_dir
elif 'Remote' in rep_type:
remote_config = getConfig(rep_type + "_Repository")
rep_host = remote_config['host']
rep_port = remote_config['port']
rep_location = rep_host + ':' + rep_port
else:
return
repository_id = rep_login + '@' + rep_location
master_job = job.master
if master_job is not None:
master_id = master_job.id
self._logger.debug('found master: %d' % master_id)
self.gangaTaskId = self.taskPrefix + '_' + \
str(master_id) + '_' + repository_id
self.gangaJobId = str(job.id)
else:
self.gangaTaskId = self.taskPrefix + \
'_' + str(job.id) + '_' + repository_id
self.gangaJobId = '0'
self._logger.debug('task_id = %s' % self.gangaTaskId)
self._logger.debug('task_job_id = %s' % self.gangaJobId)
backendConfig = getConfig(self.gridBackend)
try:
self.VO = backendConfig['VirtualOrganisation']
except KeyError:
self._logger.debug('VirtualOrganisation not configured')
# we need it, it's too dangerous if we are not sure
return
from Ganga.GPIDev.Credentials import getCredential
proxy = getCredential('GridProxy')
self.gridCertificate = proxy.info('-subject')
if self.gridCertificate is None:
self._logger.debug('error: grid certificate not known')
return
if self.gridJobId is None:
self._logger.debug('normal: grid job ID is None')
self._logger.debug('job is complete')
self._complete = True
# we can now initialize the dashboard communication thing
if self.gridJobId is not None:
try:
self.dashboard = DashboardAPI(
self.gangaTaskId, self.gangaJobId + '_' + self.gridJobId)
except TypeError:
self.dashboard = DashboardAPI(
self.gangaTaskId, self.gangaJobId + '_' + '_'.join(self.gridJobId))
