def should_wait_batch_cb(t_total, critical_thread_ids,
non_critical_thread_ids):
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import config
from Ganga.Utility.logging import getLogger
# if there are critical threads then wait or shutdown depending on
# configuration
if critical_thread_ids:
if t_total < config['forced_shutdown_timeout']:
return True
else:
getLogger().warning(
'Shutdown was forced after waiting for %d seconds for background activities to finish\
(monitoring, output download, etc). This may result in some jobs being corrupted.',
t_total)
return False
# if there are non-critical threads then wait or shutdown depending on
# configuration
elif non_critical_thread_ids:
if t_total < config['forced_shutdown_first_prompt_time']:
return True
else:
return False
# if there are no threads then shutdown
else:
return False
def fullpath(path, force=False):
"""expandfilename() and additionally: strip leading and trailing whitespaces and expand symbolic links"""
full_path = os.path.realpath(expandfilename(path.strip(), True))
if os.path.exists(full_path) or force:
return full_path
from Ganga.Utility.logging import getLogger
getLogger().debug("path: %s doesn't exist using it anyway" % path)
return path
def fullpath(path, force=False):
"""expandfilename() and additionally: strip leading and trailing whitespaces and expand symbolic links"""
full_path = os.path.realpath(expandfilename(path.strip(), True))
if os.path.exists(full_path) or force:
return full_path
from Ganga.Utility.logging import getLogger
getLogger().debug("path: %s doesn't exist using it anyway" % path)
return path
def expandfilename(filename, force=False):
"""expand a path or filename in a standard way so that it may contain ~ and ${VAR} strings"""
expanded_path = os.path.expandvars(os.path.expanduser(filename))
if os.path.exists(expanded_path) or force:
return expanded_path
from Ganga.Utility.logging import getLogger
getLogger().debug("Filename: %s doesn't exist using it anyway" % filename)
return filename
def expandfilename(filename, force=False):
"""expand a path or filename in a standard way so that it may contain ~ and ${VAR} strings"""
expanded_path = os.path.expandvars(os.path.expanduser(filename))
if os.path.exists(expanded_path) or force:
return expanded_path
from Ganga.Utility.logging import getLogger
getLogger().debug("Filename: %s doesn't exist using it anyway" % filename)
return filename
def expandfilename(filename, force=False):
"""expand a path or filename in a standard way so that it may contain ~ and ${VAR} strings"""
if filename in _stored_expanded_paths:
expanded_path = _stored_expanded_paths[filename]
else:
expanded_path = os.path.expandvars(os.path.expanduser(filename))
_stored_expanded_paths[filename] = expanded_path
if os.path.exists(expanded_path) or force:
return expanded_path
getLogger().debug("Filename: %s doesn't exist using it anyway" % filename)
return filename
def fullpath(path, force=False):
"""expandfilename() and additionally: strip leading and trailing whitespaces and expand symbolic links"""
if path in _stored_full_paths:
full_path = _stored_full_paths[path]
else:
full_path = os.path.realpath(expandfilename(path.strip(), True))
_stored_full_paths[path] = full_path
if os.path.exists(full_path) or force:
return full_path
getLogger().debug("path: %s doesn't exist using it anyway" % path)
return path
def bootstrap(reg_slice, interactive_session, my_interface=None):
"""Create local subsystems.
This function will change the default value of autostart of the monitoring, depending if the session is interactive
or batch. The autostart value may be overridden in the config file, so warn if it differs from the default.
Args:
reg_slice (RegistrySlice): A registry slice encompassing the Registry to monitor,
e.g. from getRegistrySlice('jobs') -> JobRegistry.getSlice()
interactive_session (bool): Flag indicating an interactive session or not
my_interface (Optional[module]): Public interface to add the runMonitoring function to, None (default) will set
it to Ganga.GPI
"""
# Must do some Ganga imports here to avoid circular importing
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor
from Ganga.Utility.Config import getConfig
from Ganga.Runtime.GPIexport import exportToInterface
from Ganga.Utility.logging import getLogger
global monitoring_component
# start the monitoring loop
monitoring_component = JobRegistry_Monitor(reg_slice)
monitoring_component.start()
# override the default monitoring autostart value with the setting from interactive session
config = getConfig("PollThread")
config.overrideDefaultValue('autostart', interactive_session)
# has the user changed monitoring autostart from the default? if so, warn them
if config['autostart'] != interactive_session:
if config['autostart']:
getLogger().warning(
'Monitoring loop enabled (the default setting for a batch session is disabled)'
)
else:
getLogger().warning(
'Monitoring loop disabled (the default setting for an interactive session is enabled)'
)
# Enable job monitoring if requested
if config['autostart']:
monitoring_component.enableMonitoring()
# export the runMonitoring function to the public interface
if not my_interface:
import Ganga.GPI
my_interface = Ganga.GPI
exportToInterface(my_interface, 'runMonitoring',
monitoring_component.runMonitoring, 'Functions')
def postBootstrapHook():
from Ganga.Utility.logging import getLogger
logger = getLogger()
from Ganga.Utility.Config import getConfig
cfg = getConfig('MonitoringServices')
MONITORING_DEFAULT = "Ganga.Lib.MonitoringServices.Dashboard.LCGAthenaMS.LCGAthenaMS"
for name in cfg.options:
value = cfg[name]
if 'Athena' in name.split('/') and ('LCG' in name.split('/')
or 'CREAM' in name.split('/')):
if not MONITORING_DEFAULT in value.split(','):
logger.error(
'''*** Outdated monitoring configuration - check your configuration files ***
*** Outdated monitoring configuration - check your configuration files ***
Maybe your ~/.gangarc contains old entries which override new defaults?
You may also check the configuration files defined by $GANGA_CONFIG_PATH or $GANGA_CONFIG environment variables.
To fix this problem simply remove (or comment out) the following lines in [MonitoringServices] section:
Athena/LCG=...
Athena/CREAM=...
For now I will add the correct default settings (%s) to the configuration of this Ganga session.
Note that in the future you won't be able to start Ganga until these issues are corrected manually.
''' % MONITORING_DEFAULT)
cfg.setUserValue(name, value + ',' + MONITORING_DEFAULT)
def __init__(self, repo=None, what=""):
"""
This is a fatal repo error
Args:
repo (GangaRepository): The repository the error happened in
what (str): The original exception/error/description
"""
super(RepositoryError, self).__init__(self, what)
self.what = what
self.repository = repo
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.error("A severe error occurred in the Repository '%s': %s" % (repo.registry.name, what))
logger.error('If you believe the problem has been solved, type "reactivate()" to re-enable ')
try:
from Ganga.Core.InternalServices.Coordinator import disableInternalServices
disableInternalServices()
from Ganga.Core.GangaThread.WorkerThreads import shutdownQueues
shutdownQueues()
logger.error("Shutting Down Repository_runtime")
from Ganga.Runtime import Repository_runtime
repository_runtime.shutdown()
except:
logger.error("Unable to disable Internal services, they may have already been disabled!")
def __init__(self, repo=None, what=''):
"""
This is a fatal repo error
Args:
repo (GangaRepository): The repository the error happened in
what (str): The original exception/error/description
"""
super(RepositoryError, self).__init__(self, what)
self.what = what
self.repository = repo
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.error("A severe error occurred in the Repository '%s': %s" %
(repo.registry.name, what))
logger.error(
'If you believe the problem has been solved, type "reactivate()" to re-enable '
)
try:
from Ganga.Core.InternalServices.Coordinator import disableInternalServices
disableInternalServices()
from Ganga.Core.GangaThread.WorkerThreads import shutDownQueues
shutDownQueues()
logger.error("Shutting Down Repository_runtime")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
except:
logger.error(
"Unable to disable Internal services, they may have already been disabled!"
)
def loadPlugins(environment):
"""
Given a list of environments fully load the found Plugins into them exposing all of the relavent objects
"""
from Ganga.Utility.Runtime import allRuntimes
from Ganga.Utility.logging import getLogger
logger = getLogger()
env_dict = environment.__dict__
logger.debug("Loading: %s PLUGINS" % str(allRuntimes.keys()))
for n, r in allRuntimes.iteritems():
logger.debug("Bootstrapping: %s" % n)
try:
r.bootstrap(env_dict)
except Exception as err:
logger.error('problems with bootstrapping %s -- ignored', n)
logger.error('Reason: %s' % str(err))
raise err
try:
r.loadNamedTemplates(env_dict, Ganga.Utility.Config.getConfig('Configuration')['namedTemplates_ext'],
Ganga.Utility.Config.getConfig('Configuration')['namedTemplates_pickle'])
except Exception as err:
logger.error('problems with loading Named Templates for %s', n)
logger.error('Reason: %s' % str(err))
for n, r in allRuntimes.iteritems():
logger.debug("Loading: %s" % n)
try:
r.loadPlugins()
except Exception as err:
logger.error('problems with loading Plugin %s', n)
logger.error('Reason: %s' % str(err))
raise PluginError("Failed to load plugin: %s. Ganga will now shutdown to prevent job corruption." % n)
def __handleError(self):
import sys, traceback
from Ganga.Utility.logging import getLogger
l = getLogger()
ei = sys.exc_info()
l.error(str(ei[0]) + ": " + str(ei[1]))
l.error(str(traceback.format_tb(ei[2])))
def shutdown():
# Shutdown method for all repgistries in order
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.info('Registry Shutdown')
#import traceback
#traceback.print_stack()
# Flush all repos before we shut them down
flush_all()
# shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
# method to perform actions on the box and/or job registries.
logger.debug(started_registries)
all_registries = getRegistries()
try:
if 'prep' in started_registries:
registry = getRegistry('prep')
registry.shutdown()
# in case this is called repeatedly, only call shutdown once
started_registries.remove(registry.name)
except Exception as err:
logger.error("Err: %s" % err)
logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
logger.error("Trying to shutdown cleanly regardless")
for registry in getRegistries():
thisName = registry.name
try:
if not thisName in started_registries:
continue
# in case this is called repeatedly, only call shutdown once
started_registries.remove(thisName)
registry.shutdown() # flush and release locks
except Exception as x:
logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
logger.error("%s" % x)
logger.error("Trying to Shutdown cleanly regardless")
for registry in all_registries:
my_reg = [registry]
if hasattr(registry, 'metadata'):
if registry.metadata:
my_reg.append(registry.metadata)
assigned_attrs = ['location', 'type']
for this_reg in my_reg:
for attr in assigned_attrs:
if hasattr(registry, attr):
delattr(registry, attr)
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
removeRegistries()
def emptyRepositories():
from Ganga.Utility.logging import getLogger
logger = getLogger()
# empty repository so we start again at job 0 when we restart
logger.info("Clearing the Job and Template repositories")
from Ganga.GPI import jobs, templates, tasks
for j in jobs:
try:
j.remove()
except:
pass
for t in templates:
try:
t.remove()
except:
pass
for t in tasks:
try:
t.remove(remove_jobs=True)
except:
pass
if hasattr(jobs, 'clean'):
jobs.clean(confirm=True, force=True)
if hasattr(templates, 'clean'):
templates.clean(confirm=True, force=True)
if hasattr(tasks, 'clean'):
tasks.clean(confirm=True, force=True)
def setPluginDefaults(my_interface=None):
"""
Set the plugin defaults for things like getting the defult plugin based upon class
"""
from Ganga.Utility.Plugin import allPlugins
# set the default value for the plugins
from Ganga.Utility.Config import getConfig
default_plugins_cfg = getConfig("Plugins")
from Ganga.Utility.logging import getLogger
logger = getLogger()
for opt in default_plugins_cfg:
try:
category, tag = opt.split('_')
except ValueError, err:
logger.warning("do not understand option %s in [Plugins]", opt)
logger.debug('Reason: want %s' % str(err))
else:
if tag == 'default':
try:
allPlugins.setDefault(category, default_plugins_cfg[opt])
except Ganga.Utility.Plugin.PluginManagerError as x:
logger.warning('cannot set the default plugin "%s": %s' %
(opt, x))
else:
logger.warning("do not understand option %s in [Plugins]", opt)
def __init__(self):
GangaThread.__init__(self, 'LGI_Pilot')
self.log = getLogger('LGI.Pilot.Thread')
if not os.path.exists(config['PilotScript']):
self.log.error('pilotjob script not found: '+config['PilotScript'])
if not os.path.exists(config['PilotDist']):
self.log.error('pilotjob tarball not found: '+config['PilotDist'])
def postBootstrapHook():
from Ganga.Utility.logging import getLogger
logger = getLogger()
from Ganga.Utility.Config import getConfig
cfg = getConfig('MonitoringServices')
MONITORING_DEFAULT = "Ganga.Lib.MonitoringServices.Dashboard.LCGAthenaMS.LCGAthenaMS"
for name in cfg.options:
value = cfg[name]
if 'Athena' in name.split('/') and ('LCG' in name.split('/') or 'CREAM' in name.split('/')):
if not MONITORING_DEFAULT in value.split(','):
logger.error('''*** Outdated monitoring configuration - check your configuration files ***
*** Outdated monitoring configuration - check your configuration files ***
Maybe your ~/.gangarc contains old entries which override new defaults?
You may also check the configuration files defined by $GANGA_CONFIG_PATH or $GANGA_CONFIG environment variables.
To fix this problem simply remove (or comment out) the following lines in [MonitoringServices] section:
Athena/LCG=...
Athena/CREAM=...
For now I will add the correct default settings (%s) to the configuration of this Ganga session.
Note that in the future you won't be able to start Ganga until these issues are corrected manually.
'''%MONITORING_DEFAULT)
cfg.setUserValue(name,value+','+MONITORING_DEFAULT)
def shutdown(self):
"""Shutdown the Ganga session.
@param should_wait_cb: A callback function with the following signature
should_wait_cb(total_time, critical_thread_ids, non_critical_thread_ids)
where
total_time is the time in seconds since shutdown started
critical_thread_ids is a list of alive critical thread names
non_critical_thread_ids is a list of alive non-critical threads names.
and
return value is evaluated as a boolean.
A shutdown thread is started that calls stop() on each GangaThread and
waits for them all to die. A loop waits for the shutdown thread to
die, periodically calling the should_wait_cb function to ask if it
should continue to wait or shutdown anyway.
"""
try:
self._really_shutdown()
except Exception as err:
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
logger.error("Error shutting down thread Pool!")
logger.error("\n%s" % err)
return
def __init__(self, logger=None):
if not logger:
logger = getLogger(name='Ganga.Lib.LCG.ElapsedTimeProfiler')
self.logger = logger
pass
def _store_dirac_environment():
diracversion = _guess_version('LHCBDIRAC')
import tempfile
import subprocess
import os
platform = os.environ['CMTOPT']
setup_script = 'SetupProject.sh'
env = {}
fdir = os.path.join(os.path.expanduser("~/.cache/LHCbDIRAC_ENV"), platform)
if not os.path.exists(fdir):
os.makedirs(fdir)
fname = os.path.join(fdir, diracversion)
if not os.path.exists(fname) or not os.path.getsize(fname):
from Ganga.Utility.logging import getLogger
log = getLogger()
log.info("Storing new LHCbDirac environment (%s:%s)" % (str(diracversion), str(platform)))
s_file = open(fname, 'w+')
cmd = '/usr/bin/env bash -c \"source %s LHCBDIRAC %s ROOT>& /dev/null && '\
'printenv > %s\"' % (setup_script, diracversion, fname)
rc = subprocess.Popen([cmd], shell=True).wait()
if rc != 0 or not os.path.exists(fname):
msg = '--dirac: Failed to setup Dirac version %s as obtained from project dependency.' % diracversion
raise OptionValueError(msg)
count = 0
for line in s_file.readlines():
if line.find('DIRAC') >= 0:
count += 1
varval = line.strip().split('=')
env[varval[0]] = ''.join(varval[1:])
s_file.close()
if count == 0:
msg = 'Tried to setup Dirac version %s. For some reason this did not setup the DIRAC environment.' % diracversion
raise OptionValueError(msg)
os.environ['GANGADIRACENVIRONMENT'] = fname
def getNickname(gridProxy=None,allowMissingNickname=True):
import re
from Ganga.Utility.logging import getLogger
from Ganga.GPIDev.Credentials_old import GridProxy
logger = getLogger()
if not gridProxy:
gridProxy=GridProxy()
nickName = ''
output = gridProxy.info(opt = '-all')
for line in output.split('\n'):
if line.startswith('attribute'):
match = re.search('nickname =\s*([^\s]+)\s*\(atlas\)',line)
if match != None:
nickName = match.group(1)
break
# check
if nickName == '':
from Ganga.Core.exceptions import ApplicationConfigurationError
wMessage = 'Could not get nickname from voms proxy. '
wMessage += 'Please register nickname to ATLAS VO via\n\n'
wMessage += ' https://lcg-voms.cern.ch:8443/vo/atlas/vomrs\n'
wMessage += ' [Member Info] -> [Edit Personal Info]\n\n'
wMessage += 'Then you can use the new naming convention "user.nickname" '
wMessage += 'which should be shorter than "userXY.FirstnameLastname".'
if allowMissingNickname:
logger.warning(wMessage)
else:
raise ApplicationConfigurationError(wMessage)
return nickName
class GridFileIndex(GangaObject):
'''
Data object for indexing a file on the grid.
@author: Hurng-Chun Lee
@contact: [email protected]
'''
_schema = Schema(
Version(1, 0), {
'id':
SimpleItem(defvalue='', doc='the main identity of the file'),
'name':
SimpleItem(defvalue='', doc='the name of the file'),
'md5sum':
SimpleItem(defvalue='', doc='the md5sum of the file'),
'attributes':
SimpleItem(defvalue={}, doc='a key:value pairs of file metadata')
})
_category = 'GridFileIndex'
_name = 'GridFileIndex'
logger = getLogger()
def __init__(self):
super(GridFileIndex, self).__init__()
def __eq__(self, other):
return other.id == self.id
def getNickname(gridProxy=None, allowMissingNickname=True):
import re
from Ganga.Utility.logging import getLogger
from Ganga.GPIDev.Credentials import GridProxy
logger = getLogger()
if not gridProxy:
gridProxy = GridProxy()
nickName = ''
output = gridProxy.info(opt='-all')
for line in output.split('\n'):
if line.startswith('attribute'):
match = re.search('nickname =\s*([^\s]+)\s*\(atlas\)', line)
if match != None:
nickName = match.group(1)
break
# check
if nickName == '':
from Ganga.Core.exceptions import ApplicationConfigurationError
wMessage = 'Could not get nickname from voms proxy. '
wMessage += 'Please register nickname to ATLAS VO via\n\n'
wMessage += ' https://lcg-voms.cern.ch:8443/vo/atlas/vomrs\n'
wMessage += ' [Member Info] -> [Edit Personal Info]\n\n'
wMessage += 'Then you can use the new naming convention "user.nickname" '
wMessage += 'which should be shorter than "userXY.FirstnameLastname".'
if allowMissingNickname:
logger.warning(wMessage)
else:
raise ApplicationConfigurationError(None, wMessage)
return nickName
def shutdown(self):
"""Flush and disconnect the repository. Called from Repository_runtime.py """
from Ganga.Utility.logging import getLogger
logger = getLogger()
self.shareref = self.metadata[self.metadata.ids()[-1]]
self._lock.acquire()
#self.shareref.closedown() ## Commenting out a potentially EXTREMELY heavy operation from shutdown after ganga dev meeting - rcurrie
try:
try:
if not self.metadata is None:
self.metadata.shutdown()
except Exception as x:
logger.error("Exception on shutting down metadata repository '%s' registry: %s", self.name, x)
try:
self._flush()
except Exception as x:
logger.error("Exception on flushing '%s' registry: %s", self.name, x)
self._started = False
for obj in self._objects.values():
# locks are not guaranteed to survive repository shutdown
obj._registry_locked = False
self.repository.shutdown()
except Exception as err:
logger.debug("Shutdown Error: %s" % str(err))
finally:
self._lock.release()
def _store_dirac_environment():
diracversion = _guess_version('LHCBDIRAC')
import tempfile
import subprocess
import os
platform = os.environ['CMTOPT']
setup_script = 'SetupProject.sh'
env = {}
fdir = os.path.join(os.path.expanduser("~/.cache/LHCbDIRAC_ENV"), platform)
if not os.path.exists(fdir):
os.makedirs(fdir)
fname = os.path.join(fdir, diracversion)
if not os.path.exists(fname) or not os.path.getsize(fname):
from Ganga.Utility.logging import getLogger
log = getLogger()
log.info("Storing new LHCbDirac environment (%s:%s)" %
(str(diracversion), str(platform)))
s_file = open(fname, 'w+')
cmd = '/usr/bin/env bash -c \"source %s LHCBDIRAC %s ROOT>& /dev/null && '\
'printenv > %s\"' % (setup_script, diracversion, fname)
rc = subprocess.Popen([cmd], shell=True).wait()
if rc != 0 or not os.path.exists(fname):
msg = '--dirac: Failed to setup Dirac version %s as obtained from project dependency.' % diracversion
raise OptionValueError(msg)
count = 0
for line in s_file.readlines():
if line.find('DIRAC') >= 0:
count += 1
varval = line.strip().split('=')
env[varval[0]] = ''.join(varval[1:])
s_file.close()
if count == 0:
msg = 'Tried to setup Dirac version %s. For some reason this did not setup the DIRAC environment.' % diracversion
raise OptionValueError(msg)
os.environ['GANGADIRACENVIRONMENT'] = fname
def shutdown(self, should_wait_cb=None):
"""Shutdown the Ganga session.
@param should_wait_cb: A callback function with the following signature
should_wait_cb(total_time, critical_thread_ids, non_critical_thread_ids)
where
total_time is the time in seconds since shutdown started
critical_thread_ids is a list of alive critical thread names
non_critical_thread_ids is a list of alive non-critical threads names.
and
return value is evaluated as a boolean.
A shutdown thread is started that calls stop() on each GangaThread and
waits for them all to die. A loop waits for the shutdown thread to
die, periodically calling the should_wait_cb function to ask if it
should continue to wait or shutdown anyway.
"""
try:
self._really_shutdown(should_wait_cb)
except Exception as err:
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
logger.error("Error shutting down thread Pool!")
logger.error("\n%s" % str(err))
return
def loadPlugins(environment):
"""
Given a list of environments fully load the found Plugins into them exposing all of the relavent objects
"""
from Ganga.Utility.Runtime import allRuntimes
from Ganga.Utility.logging import getLogger
logger = getLogger()
env_dict = environment.__dict__
for n, r in zip(allRuntimes.keys(), allRuntimes.values()):
try:
r.bootstrap(env_dict)
except Exception as err:
logger.error('problems with bootstrapping %s -- ignored', n)
logger.error('Reason: %s' % str(err))
raise err
try:
r.loadNamedTemplates(env_dict, Ganga.Utility.Config.getConfig('Configuration')['namedTemplates_ext'],
Ganga.Utility.Config.getConfig('Configuration')['namedTemplates_pickle'])
except Exception as err:
logger.error('problems with loading Named Templates for %s', n)
logger.error('Reason: %s' % str(err))
for r in allRuntimes.values():
try:
r.loadPlugins()
except Exception as err:
logger.error('problems with loading Named Templates for %s', n)
logger.error('Reason: %s' % str(err))
def __init__(self,
name,
algorithm=None,
data=None,
numThread=10,
keepAlive=False):
"""
initializes the MTRunner object.
@since: 0.0.1
@author: Hurng-Chun Lee
@contact: [email protected]
@param algorithm is an Algorithm object defining how to process on the data
@param data is an Data object defining what to be processed by the algorithm
"""
if (not algorithm) or (not data):
raise MTRunnerError('algorithm and data must not be None')
self.algorithm = algorithm
self.data = data
self.numThread = numThread
self.doneList = []
self.lock = Lock()
self.name = name
self.keepAlive = keepAlive
self._agents = []
self.logger = getLogger()
def emptyRepositories():
"""
A method which attempts to remove jobs from various repositories in a sane manner,
This is preferred to just shutting down and runnning rm -fr ... as it catches a few errors hard to test for
"""
from Ganga.Utility.logging import getLogger
logger = getLogger()
# empty repository so we start again at job 0 when we restart
logger.info("Clearing the Job and Template repositories")
from Ganga.GPI import jobs, templates, tasks
for j in jobs:
try:
j.remove()
except:
pass
for t in templates:
try:
t.remove()
except:
pass
for t in tasks:
try:
t.remove(remove_jobs=True)
except:
pass
if hasattr(jobs, 'clean'):
jobs.clean(confirm=True, force=True)
if hasattr(templates, 'clean'):
templates.clean(confirm=True, force=True)
if hasattr(tasks, 'clean'):
tasks.clean(confirm=True, force=True)
def shutdown(self):
"""Flush and disconnect the repository. Called from Repository_runtime.py """
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.debug("Shutting Down Registry")
logger.debug("shutdown")
try:
self._hasStarted = True
# NB flush_all by definition relies on both the metadata repo and the repo to be fully initialized
try:
self.flush_all()
except Exception as err:
logger.error("Exception on flushing '%s' registry: %s", self.name, err)
# Now we can safely shutdown the metadata repo if one is loaded
try:
if self.metadata is not None:
self.metadata.shutdown()
except Exception as err:
logger.debug("Exception on shutting down metadata repository '%s' registry: %s", self.name, err)
raise
# Now we can release locks on the objects we have
for obj in self._objects.values():
# locks are not guaranteed to survive repository shutdown
obj._registry_locked = False
self.repository.shutdown()
self.metadata = None
finally:
self._hasStarted = False
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('ARC bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def shutdown(self):
"""Flush and disconnect the repository. Called from Repository_runtime.py """
self._hasStarted = True
from Ganga.Utility.logging import getLogger
self.shouldRun = True
## Aparently this shuts down the metadata repo before we want to shut it down...
#super(PrepRegistry, self).shutdown()
logger = getLogger()
#logger.info("Geting id: %s" % self.metadata.ids()[-1])
self.shareref = self.metadata._objects[self.metadata.ids()[-1]]
#logger.info("ShareRef: %s" % getName(self.shareref))
with self._flush_lock:
with self._read_lock:
## THIS IS DISABLED AS IT REQUIRES ACCESS TO REPO OBJECTS THROUGH GETREADACCES...
## THIS NEEDS TO BE FIXED OR IMPLEMENTED AS A SHUTDOWN SERVICE!!!
try:
stripProxy(self.shareref).closedown(
) ## Commenting out a potentially EXTREMELY heavy operation from shutdown after ganga dev meeting - rcurrie
except Exception as err:
logger.error("Shutdown Error in ShareRef")
logger.error("Err: %s" % err)
try:
self._safe_shutdown()
except Exception as err:
logger.debug("Shutdown Error: %s" % err)
self._hasStarted = False
self.metadata = None
def get_md5sum(fname, ignoreGzipTimestamp=False):
''' Calculates the MD5 checksum of a file '''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
## if the file is a zipped format (determined by extension),
## try to get checksum from it's content. The reason is that
## gzip file contains a timestamp in the header, which causes
## different md5sum value even the contents are the same.
#re_gzipfile = re.compile('.*[\.tgz|\.gz].*$')
f = None
if ignoreGzipTimestamp and (fname.find('.tgz') > 0 or fname.find('.gz') > 0):
f = gzip.open(fname,'rb')
else:
f = open(fname, 'rb')
m = get_md5_obj()
while True:
d = f.read(8096)
if not d:
break
m.update(d)
f.close()
md5sum = m.hexdigest()
profiler.check('md5sum calculation time')
return md5sum
def select(self, minid=None, maxid=None, **attrs):
import repr
from Ganga.GPIDev.Lib.Job.Job import Job
if isType(minid, Job):
if minid.master:
minid = minid.master.id
else:
minid = minid.id
if maxid is None:
maxid = minid
if isType(maxid, Job):
if maxid.master:
maxid = maxid.master.id
else:
maxid = maxid.id
logger = getLogger()
this_repr = repr.Repr()
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
attrs_str = ""
## Loop through all possible input combinations to constructa string representation of the attrs from possible inputs
## Reuired to flatten the additional arguments into a flat string in attrs_str
for a in attrs:
from inspect import isclass
if isclass(attrs[a]):
this_attr = GPIProxyObjectFactory(attrs[a]())
else:
from Ganga.GPIDev.Base.Objects import GangaObject
if isType(attrs[a], GangaObject):
this_attr = GPIProxyObjectFactory(attrs[a])
else:
if type(attrs[a]) is str:
from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject
this_attr = getRuntimeGPIObject(attrs[a], True)
else:
this_attr = attrs[a]
full_str = str(this_attr)
split_str = full_str.split('\n')
for line in split_str:
line = line.strip()
flat_str = ''.join(split_str)
attrs_str += ", %s=\"%s\"" % (str(a), flat_str)
logger.debug("Attrs_Str: %s" % str(attrs_str))
logger.debug("Constructing slice: %s" %
str("%s.select(minid='%s', maxid='%s'%s)" %
(self.name, this_repr.repr(minid),
this_repr.repr(maxid), attrs_str)))
this_slice = self.__class__("%s.select(minid='%s', maxid='%s'%s)" %
(self.name, this_repr.repr(minid),
this_repr.repr(maxid), attrs_str))
def append(id, obj):
this_slice.objects[id] = obj
self.do_select(append, minid, maxid, **attrs)
return this_slice
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('ARC bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def emptyRepositories():
"""
A method which attempts to remove jobs from various repositories in a sane manner,
This is preferred to just shutting down and runnning rm -fr ... as it catches a few errors hard to test for
"""
from Ganga.Utility.logging import getLogger
logger = getLogger()
# empty repository so we start again at job 0 when we restart
logger.info("Clearing the Job and Template repositories")
from Ganga.GPI import jobs, templates, tasks
for j in jobs:
try:
j.remove()
except:
pass
for t in templates:
try:
t.remove()
except:
pass
for t in tasks:
try:
t.remove(remove_jobs=True)
except:
pass
if hasattr(jobs, 'clean'):
jobs.clean(confirm=True, force=True)
if hasattr(templates, 'clean'):
templates.clean(confirm=True, force=True)
if hasattr(tasks, 'clean'):
tasks.clean(confirm=True, force=True)
def loadPlugins(environment):
"""
Given a list of environments fully load the found Plugins into them exposing all of the relavent objects
"""
from Ganga.Utility.Runtime import allRuntimes
from Ganga.Utility.logging import getLogger
logger = getLogger()
env_dict = environment.__dict__
for n, r in allRuntimes.iteritems():
try:
r.bootstrap(env_dict)
except Exception as err:
logger.error('problems with bootstrapping %s -- ignored', n)
logger.error('Reason: %s' % str(err))
raise err
try:
r.loadNamedTemplates(env_dict, Ganga.Utility.Config.getConfig('Configuration')['namedTemplates_ext'],
Ganga.Utility.Config.getConfig('Configuration')['namedTemplates_pickle'])
except Exception as err:
logger.error('problems with loading Named Templates for %s', n)
logger.error('Reason: %s' % str(err))
for n, r in allRuntimes.iteritems():
try:
r.loadPlugins()
except Exception as err:
logger.error('problems with loading Plugin %s', n)
logger.error('Reason: %s' % str(err))
raise PluginError("Failed to load plugin: %s. Ganga will now shutdown to prevent job corruption." % n)
def shutdown(self):
"""Flush and disconnect the repository. Called from Repository_runtime.py """
from Ganga.Utility.logging import getLogger
self.shouldRun = True
## Aparently this shuts down the metadata repo before we want to shut it down...
#super(PrepRegistry, self).shutdown()
logger = getLogger()
self.shareref = self.metadata[self.metadata.ids()[-1]]
self._lock.acquire()
## THIS IS DISABLED AS IT REQUIRES ACCESS TO REPO OBJECTS THROUGH GETREADACCES...
## THIS NEEDS TO BE FIXED OR IMPLEMENTED AS A SHUTDOWN SERVICE!!!
try:
self.shareref.closedown() ## Commenting out a potentially EXTREMELY heavy operation from shutdown after ganga dev meeting - rcurrie
except Exception as err:
logger.error("Shutdown Error in ShareRef")
logger.error("Err: %s" % str(err))
try:
try:
if not self.metadata is None:
self.metadata.shutdown()
except Exception as x:
logger.error("Exception on shutting down metadata repository '%s' registry: %s", self.name, x)
try:
self._flush()
except Exception as x:
logger.error("Exception on flushing '%s' registry: %s", self.name, x)
self._started = False
for obj in self._objects.values():
# locks are not guaranteed to survive repository shutdown
obj._registry_locked = False
self.repository.shutdown()
except Exception as err:
logger.debug("Shutdown Error: %s" % str(err))
finally:
self._lock.release()
def shutdown(self):
"""Flush and disconnect the repository. Called from Repository_runtime.py """
self._hasStarted = True
from Ganga.Utility.logging import getLogger
self.shouldRun = True
## Aparently this shuts down the metadata repo before we want to shut it down...
#super(PrepRegistry, self).shutdown()
logger = getLogger()
#logger.info("Geting id: %s" % self.metadata.ids()[-1])
self.shareref = self.metadata._objects[self.metadata.ids()[-1]]
#logger.info("ShareRef: %s" % getName(self.shareref))
self._lock.acquire()
## THIS IS DISABLED AS IT REQUIRES ACCESS TO REPO OBJECTS THROUGH GETREADACCES...
## THIS NEEDS TO BE FIXED OR IMPLEMENTED AS A SHUTDOWN SERVICE!!!
try:
stripProxy(self.shareref).closedown() ## Commenting out a potentially EXTREMELY heavy operation from shutdown after ganga dev meeting - rcurrie
except Exception as err:
logger.error("Shutdown Error in ShareRef")
logger.error("Err: %s" % str(err))
try:
self._safe_shutdown()
except Exception as err:
logger.debug("Shutdown Error: %s" % str(err))
finally:
self._hasStarted = False
self._lock.release()
def get_md5sum(fname, ignoreGzipTimestamp=False):
''' Calculates the MD5 checksum of a file '''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
# if the file is a zipped format (determined by extension),
# try to get checksum from it's content. The reason is that
# gzip file contains a timestamp in the header, which causes
# different md5sum value even the contents are the same.
#re_gzipfile = re.compile('.*[\.tgz|\.gz].*$')
f = None
if ignoreGzipTimestamp and (fname.find('.tgz') > 0
or fname.find('.gz') > 0):
f = gzip.open(fname, 'rb')
else:
f = open(fname, 'rb')
m = hashlib.md5()
while True:
d = f.read(8096)
if not d:
break
m.update(d)
f.close()
md5sum = m.hexdigest()
profiler.check('md5sum calculation time')
return md5sum
def stop_ganga():
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.info("Deciding how to shutdown")
# Do we want to empty the repository on shutdown?
from Ganga.Utility.Config import getConfig
if 'AutoCleanup' in getConfig('TestingFramework'):
whole_cleanup = getConfig('TestingFramework')['AutoCleanup']
else:
whole_cleanup = True
logger.info("AutoCleanup: %s" % whole_cleanup)
if whole_cleanup is True:
# empty repository so we start again at job 0 when we restart
logger.info("Clearing the Job and Template repositories")
from Ganga.GPI import jobs, templates
for j in jobs:
try:
j.remove()
except:
pass
for t in templates:
try:
t.remove()
except:
pass
if hasattr(jobs, 'clean'):
jobs.clean(confirm=True, force=True)
if hasattr(templates, 'clean'):
templates.clean(confirm=True, force=True)
logger.info("Shutting Down Internal Services")
# Disable internal services such as monitoring and other tasks
#from Ganga.Core.InternalServices import Coordinator
# if Coordinator.servicesEnabled:
# Coordinator.disableInternalServices()
# Coordinator.servicesEnabled = False
logger.info("Mimicking ganga exit")
from Ganga.Core.InternalServices import ShutdownManager
import Ganga.Core
Ganga.Core.change_atexitPolicy(interactive_session=False, new_policy='batch')
# This should now be safe
ShutdownManager._ganga_run_exitfuncs()
# Undo any manual editing of the config and revert to defaults
from Ganga.Utility.Config import allConfigs
for package in allConfigs.values():
package.revertToDefaultOptions()
# Finished
logger.info("Test Finished")
def bootstrap(reg_slice, interactive_session, my_interface=None):
"""
Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
FIXME: this procedure should be moved to the Runtime package.
This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
The autostart value may be overriden in the config file, so warn if it differs from the default.
"""
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config
from Ganga.Utility.logging import getLogger
logger = getLogger()
from Ganga.Core.GangaThread import GangaThreadPool
# start the internal services coordinator
from Ganga.Core.InternalServices import Coordinator
Coordinator.bootstrap()
# backend-specific setup (e.g. Remote: setup any remote ssh pipes)
# for j in reg_slice:
# if hasattr(j,'status') and j.status in ['submitted','running']:
# if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
# if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
# j.backend.setup()
start_jobregistry_monitor(reg_slice)
# Set the shutdown policy depending on whether we're interactive or not
if config['forced_shutdown_policy'] in ['interactive', 'batch']:
GangaThreadPool.shutdown_policy = config['forced_shutdown_policy']
else:
if interactive_session:
GangaThreadPool.shutdown_policy = 'interactive'
else:
GangaThreadPool.shutdown_policy = 'batch'
# export to GPI moved to Runtime bootstrap
autostart_default = interactive_session
config.overrideDefaultValue('autostart', bool(autostart_default))
if config['autostart'] is not autostart_default:
msg = 'monitoring loop %s (the default setting for %s session is %s)'
val = {
True: ('enabled', 'batch', 'disabled'),
False: ('disabled', 'interactive', 'enabled')
}
logger.warning(msg % val[config['autostart']])
if config['autostart']:
monitoring_component.enableMonitoring()
if not my_interface:
import Ganga.GPI
my_interface = Ganga.GPI
from Ganga.Runtime.GPIexport import exportToInterface
exportToInterface(my_interface, 'runMonitoring',
monitoring_component.runMonitoring, 'Functions')
def _really_shutdown(self, should_wait_cb=None):
from Ganga.Utility.logging import getLogger
logger = getLogger('GangaThread')
logger.debug('shutting down GangaThreadPool with timeout %d sec' % self.SHUTDOWN_TIMEOUT)
# run shutdown thread in background
import threading
shutdown_thread = threading.Thread(target=self.__do_shutdown__, args=(self.__threads,), name='GANGA_Update_Thread_shutdown')
shutdown_thread.setDaemon(True)
shutdown_thread.start()
t_start = time.time()
def __cnt_alive_threads__(_all_threads):
num_alive_threads = 0
for t in _all_threads:
if t.isAlive():
num_alive_threads += 1
return num_alive_threads
# wait for the background shutdown thread to finish
while shutdown_thread.isAlive():
logger.debug('Waiting for max %d seconds for threads to finish' % self.SHUTDOWN_TIMEOUT)
logger.debug('There are %d alive background threads' % __cnt_alive_threads__(self.__threads))
logger.debug('%s' % self.__alive_critical_thread_ids())
logger.debug('%s' % self.__alive_non_critical_thread_ids())
shutdown_thread.join(self.SHUTDOWN_TIMEOUT)
if shutdown_thread.isAlive():
# if should_wait_cb callback exists then ask if we should wait
if should_wait_cb:
total_time = time.time() - t_start
critical_thread_ids = self.__alive_critical_thread_ids()
non_critical_thread_ids = self.__alive_non_critical_thread_ids()
if not should_wait_cb(total_time, critical_thread_ids, non_critical_thread_ids):
logger.debug('GangaThreadPool shutdown anyway after %d sec.' % (time.time() - t_start))
break
else:
pass
else:
logger.debug('GangaThreadPool shutdown properly')
# log warning message if critical thread still alive
critical_thread_ids = self.__alive_critical_thread_ids()
if critical_thread_ids:
logger.warning('Shutdown forced. %d background thread(s) still running: %s', len(critical_thread_ids), critical_thread_ids)
# log debug message if critical thread still alive
non_critical_thread_ids = self.__alive_non_critical_thread_ids()
if non_critical_thread_ids:
logger.debug('Shutdown forced. %d non-critical background thread(s) still running: %s', len(non_critical_thread_ids), non_critical_thread_ids)
# set singleton instance to None
self._instance = None
for i in self.__threads:
del i
self.__threads = []
def createPublisher(server, port, user="******", password="******", idle_timeout=None, exit_timeout=None):
"""Create a new publisher thread which extends GangaThread where available
(i.e. on the client) or Thread otherwise (i.e. on the worker node).
N.B. If GangaThread is not available then an exit handler is added, with the
given timeout.
@param server: The server host name.
@param user: The user name.
@param password: The password.
@param logger: The logger instance.
@param idle_timeout: Maximum seconds to idle before closing connection.
Negative value indicates never close connection.
@param exit_timeout: Maximum seconds to clear message queue on exit.
Negative value indicates clear queue without timeout.
Usage::
from Ganga.Lib.MonitoringServices.MSG import MSGUtil
p = MSGUTIL.createPublisher('dashb-mb.cern.ch', 61113)
p.start()
p.send('/topic/ganga.dashboard.test', 'Hello World!')
See also stomputil.publisher
"""
from Ganga.Utility.stomputil.publisher import IDLE_TIMEOUT, EXIT_TIMEOUT
if idle_timeout is None:
idle_timeout = IDLE_TIMEOUT
if exit_timeout is None:
exit_timeout = EXIT_TIMEOUT
# use GangaThread class on client or Thread class otherwise
try:
from Ganga.Core.GangaThread import GangaThread as Thread
managed_thread = True
except ImportError:
from threading import Thread
managed_thread = False
# use Ganga logger class on client or None otherwise
try:
from Ganga.Utility.logging import getLogger
logger = getLogger()
except ImportError:
logger = None
# create and start _publisher
import Ganga.Utility.stomputil
publisher = Ganga.Utility.stomputil.createPublisher(Thread, server, port, user, password, logger, idle_timeout)
if managed_thread:
# set GangaThread as non-critical
publisher.setCritical(False)
else:
# add exit handler if not GangaThread
publisher.addExitHandler(exit_timeout)
return publisher
def shutdown(self):
g = importName(self.name, 'shutdown')
if g:
g()
else:
from Ganga.Utility.logging import getLogger
logger = getLogger(modulename=1)
logger.debug("no shutdown procedure in runtime package %s", self.name)
def license():
'Print the full license (GPL)'
from Ganga.Utility.logging import getLogger
logger = getLogger()
from os import path
from Ganga import _gangaPythonPath
with open(path.join(_gangaPythonPath, '..', 'LICENSE_GPL')) as printable:
logger.info(printable.read())
def shutdown(self):
g = importName(self.name, 'shutdown')
if g:
g()
else:
from Ganga.Utility.logging import getLogger
logger = getLogger(modulename=1)
logger.debug("no shutdown procedure in runtime package %s", self.name)
def shutdown(self):
"""Shutdown the repository. Flushing is done by the Registry
Raise RepositoryError"""
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.debug("Shutting Down GangaRepositoryLocal: %s" % self.registry.name)
self._write_master_cache()
self.sessionlock.shutdown()
def license():
'Print the full license (GPL)'
from Ganga.Utility.logging import getLogger
logger = getLogger()
from os import path
from Ganga import _gangaPythonPath
with open(path.join(_gangaPythonPath, '..', 'LICENSE_GPL')) as printable:
logger.info(printable.read())
def bootstrap(reg_slice, interactive_session, my_interface=None):
"""
Create local subsystems. In the future this procedure should be enhanced to connect to remote subsystems.
FIXME: this procedure should be moved to the Runtime package.
This function will change the default value of autostart of the monitoring, depending if the session is interactive or batch.
The autostart value may be overriden in the config file, so warn if it differs from the default.
"""
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor, config
from Ganga.Utility.logging import getLogger
logger = getLogger()
from Ganga.Core.GangaThread import GangaThreadPool
# start the internal services coordinator
from Ganga.Core.InternalServices import Coordinator
Coordinator.bootstrap()
# backend-specific setup (e.g. Remote: setup any remote ssh pipes)
# for j in reg_slice:
# if hasattr(j,'status') and j.status in ['submitted','running']:
# if hasattr(j,'backend'): # protect: EmptyGangaObject does not have backend either
# if hasattr(j.backend,'setup'): # protect: EmptyGangaObject does not have setup() method
# j.backend.setup()
start_jobregistry_monitor(reg_slice)
# Set the shutdown policy depending on whether we're interactive or not
if config["forced_shutdown_policy"] in ["interactive", "batch"]:
GangaThreadPool.shutdown_policy = config["forced_shutdown_policy"]
else:
if interactive_session:
GangaThreadPool.shutdown_policy = "interactive"
else:
GangaThreadPool.shutdown_policy = "batch"
# export to GPI moved to Runtime bootstrap
autostart_default = interactive_session
config.overrideDefaultValue("autostart", bool(autostart_default))
if config["autostart"] is not autostart_default:
msg = "monitoring loop %s (the default setting for %s session is %s)"
val = {True: ("enabled", "batch", "disabled"), False: ("disabled", "interactive", "enabled")}
logger.warning(msg % val[config["autostart"]])
if config["autostart"]:
monitoring_component.enableMonitoring()
if not my_interface:
import Ganga.GPI
my_interface = Ganga.GPI
from Ganga.Runtime.GPIexport import exportToInterface
exportToInterface(my_interface, "runMonitoring", monitoring_component.runMonitoring, "Functions")
def bootstrap(reg_slice, interactive_session, my_interface=None):
"""Create local subsystems.
This function will change the default value of autostart of the monitoring, depending if the session is interactive
or batch. The autostart value may be overridden in the config file, so warn if it differs from the default.
Args:
reg_slice (RegistrySlice): A registry slice encompassing the Registry to monitor,
e.g. from getRegistrySlice('jobs') -> JobRegistry.getSlice()
interactive_session (bool): Flag indicating an interactive session or not
my_interface (Optional[module]): Public interface to add the runMonitoring function to, None (default) will set
it to Ganga.GPI
"""
# Must do some Ganga imports here to avoid circular importing
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import JobRegistry_Monitor
from Ganga.Utility.Config import getConfig
from Ganga.Runtime.GPIexport import exportToInterface
from Ganga.Utility.logging import getLogger
global monitoring_component
# start the monitoring loop
monitoring_component = JobRegistry_Monitor(reg_slice)
monitoring_component.start()
# override the default monitoring autostart value with the setting from interactive session
config = getConfig("PollThread")
config.overrideDefaultValue('autostart', interactive_session)
# has the user changed monitoring autostart from the default? if so, warn them
if config['autostart'] != interactive_session:
if config['autostart']:
getLogger().warning('Monitoring loop enabled (the default setting for a batch session is disabled)')
else:
getLogger().warning('Monitoring loop disabled (the default setting for an interactive session is enabled)')
# Enable job monitoring if requested
if config['autostart']:
monitoring_component.enableMonitoring()
# export the runMonitoring function to the public interface
if not my_interface:
import Ganga.GPI
my_interface = Ganga.GPI
exportToInterface(my_interface, 'runMonitoring', monitoring_component.runMonitoring, 'Functions')
def shutdown(self):
"""Shutdown the repository. Flushing is done by the Registry
Raise RepositoryError"""
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.debug("Shutting Down GangaRepositoryLocal: %s" %
self.registry.name)
self._write_master_cache(True)
self.sessionlock.shutdown()
def __init__(self, _id, repo):
self.id = _id
self.repo = repo
self.rng = random.Random()
self.owned_ids = []
self.done = False
from Ganga.Utility.logging import getLogger
self.logger = getLogger(modulename=True)
super(HammerThread, self).__init__()
def job_finalisation_cleanup(job, updated_dirac_status):
logger = getLogger()
# Revert job back to running state if we exit uncleanly
if job.status == "completing":
job.updateStatus('running')
if job.master:
job.master.updateMasterJobStatus()
def __init__(self, _id, reg):
self.id = _id
self.reg = reg
self.rng = random.Random()
self.owned_ids = []
self.owned_objs = {}
self.done = False
super(HammerThread, self).__init__() #'HammerThread_%s' % _id)
from Ganga.Utility.logging import getLogger
self.logger = getLogger(modulename=True)
def createPublisher(server, port, user='******', password='******', idle_timeout=None, exit_timeout=None):
"""Create a new publisher thread which extends GangaThread where available
(i.e. on the client) or Thread otherwise (i.e. on the worker node).
N.B. If GangaThread is not available then an exit handler is added, with the
given timeout.
@param server: The server host name.
@param user: The user name.
@param password: The password.
@param logger: The logger instance.
@param idle_timeout: Maximum seconds to idle before closing connection.
Negative value indicates never close connection.
@param exit_timeout: Maximum seconds to clear message queue on exit.
Negative value indicates clear queue without timeout.
Usage::
from Ganga.Lib.MonitoringServices.MSG import MSGUtil
p = MSGUTIL.createPublisher('dashb-mb.cern.ch', 61113)
p.start()
p.send('/topic/ganga.dashboard.test', 'Hello World!')
See also stomputil.publisher
"""
from Ganga.Utility.stomputil.publisher import IDLE_TIMEOUT, EXIT_TIMEOUT
if idle_timeout is None:
idle_timeout = IDLE_TIMEOUT
if exit_timeout is None:
exit_timeout = EXIT_TIMEOUT
# use GangaThread class on client or Thread class otherwise
try:
from Ganga.Core.GangaThread import GangaThread as Thread
managed_thread = True
except ImportError:
from threading import Thread
managed_thread = False
# use Ganga logger class on client or None otherwise
try:
from Ganga.Utility.logging import getLogger
logger = getLogger()
except ImportError:
logger = None
# create and start _publisher
import Ganga.Utility.stomputil
publisher = Ganga.Utility.stomputil.createPublisher(Thread, server, port, user, password, logger, idle_timeout)
if managed_thread:
# set GangaThread as non-critical
publisher.setCritical(False)
else:
# add exit handler if not GangaThread
publisher.addExitHandler(exit_timeout)
return publisher
