def startUpRegistries(my_interface=None):
# Startup the registries and export them to the GPI, also add jobtree and shareref
from Ganga.Runtime.GPIexport import exportToInterface
if not my_interface:
import Ganga.GPI
my_interface = Ganga.GPI
# import default runtime modules
global _runtime_interface
_runtime_interface = my_interface
# bootstrap user-defined runtime modules and enable transient named
# template registries
# bootstrap runtime modules
from Ganga.GPIDev.Lib.JobTree import TreeError
for n, k, d in bootstrap():
# make all repository proxies visible in GPI
exportToInterface(my_interface, n, k, 'Objects', d)
# JobTree
from Ganga.Core.GangaRepository import getRegistry
jobtree = getRegistry("jobs").getJobTree()
exportToInterface(my_interface, 'jobtree', jobtree, 'Objects', 'Logical tree view of the jobs')
exportToInterface(my_interface, 'TreeError', TreeError, 'Exceptions')
# ShareRef
shareref = getRegistry("prep").getShareRef()
exportToInterface(my_interface, 'shareref', shareref, 'Objects', 'Mechanism for tracking use of shared directory resources')
def startUpRegistries(my_interface=None):
# Startup the registries and export them to the GPI, also add jobtree and shareref
from Ganga.Runtime.GPIexport import exportToInterface
if not my_interface:
import Ganga.GPI
my_interface = Ganga.GPI
# import default runtime modules
global _runtime_interface
_runtime_interface = my_interface
# bootstrap user-defined runtime modules and enable transient named
# template registries
# bootstrap runtime modules
from Ganga.GPIDev.Lib.JobTree import TreeError
for n, k, d in bootstrap():
# make all repository proxies visible in GPI
exportToInterface(my_interface, n, k, 'Objects', d)
# JobTree
from Ganga.Core.GangaRepository import getRegistry
jobtree = getRegistry("jobs").getJobTree()
exportToInterface(my_interface, 'jobtree', jobtree, 'Objects',
'Logical tree view of the jobs')
exportToInterface(my_interface, 'TreeError', TreeError, 'Exceptions')
# ShareRef
shareref = getRegistry("prep").getShareRef()
exportToInterface(
my_interface, 'shareref', shareref, 'Objects',
'Mechanism for tracking use of shared directory resources')
def _copy(self, unprepare=None):
global proxyRef
logger.debug("unprepare is %s", str(unprepare))
if unprepare is None:
if prepconfig["unprepare_on_copy"] is True:
if hasattr(self, "is_prepared") or hasattr(self, "application"):
unprepare = True
def _getSharedPath():
Config_conf = getConfig("Configuration")
return os.path.join(expandfilename(Config_conf["gangadir"]), "shared", Config_conf["user"])
if hasattr(self, "application"):
if hasattr(self.application, "is_prepared"):
from Ganga.Utility.files import expandfilename
if self.application.is_prepared not in [None, True]:
if hasattr(self.application.is_prepared, "name"):
shared_path = _getSharedPath()
if os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)):
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug("increasing counter from proxy.py")
shareref.increase(self.application.is_prepared.name)
logger.debug("Found ShareDir directory: %s" % self.application.is_prepared.name)
elif self.application.is_prepared not in [None, True]:
shared_path = _getSharedPath()
if not os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)):
logger.error("ShareDir directory not found: %s" % self.application.is_prepared.name)
logger.error("Unpreparing Job #%s" % self.id)
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.application.is_prepared.name)
self.unprepare()
if unprepare is True:
if hasattr(self, "is_prepared"):
from Ganga.Utility.files import expandfilename
if self.is_prepared not in [None, True]:
if hasattr(self.is_prepared, "name"):
shared_path = _getSharedPath()
if not os.path.isdir(os.path.join(shared_path, self.is_prepared.name)):
logger.error("ShareDir directory not found: %s" % self.is_prepared.name)
logger.error("Unpreparing %s application" % getattr(self, proxyRef)._name)
self.unprepare()
c = getattr(self, proxyRef).clone()
if hasattr(c, "is_prepared") and c._getRegistry() is None:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.is_prepared.name)
c._auto__init__(unprepare=True)
else:
c = getattr(self, proxyRef).clone()
c._auto__init__()
return GPIProxyObjectFactory(c)
def add(self, input):
from Ganga.Core.GangaRepository import getRegistry
if not isType(input, list):
input = [input]
for item in input:
if isType(item, str):
if os.path.isfile(expandfilename(item)):
logger.info('Copying file %s to shared directory %s' %
(item, self.name))
shutil.copy2(expandfilename(item),
os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item))
elif isType(item, File) and item.name is not '' and os.path.isfile(
expandfilename(item.name)):
logger.info('Copying file object %s to shared directory %s' %
(item.name, self.name))
shutil.copy2(expandfilename(item.name),
os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item.name))
def _copy(self, unprepare=None):
logger.debug('unprepare is %s', str(unprepare))
if unprepare is None:
if prepconfig['unprepare_on_copy'] is True:
if hasattr(self, 'is_prepared') or hasattr(self, 'application'):
unprepare = True
def _getSharedPath():
Config_conf = getConfig('Configuration')
return os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user'])
if hasattr(self, 'application'):
if hasattr(self.application, 'is_prepared'):
from Ganga.Utility.files import expandfilename
if self.application.is_prepared not in [None, True]:
if hasattr(self.application.is_prepared, 'name'):
shared_path = _getSharedPath()
if os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)):
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug('increasing counter from proxy.py')
shareref.increase(self.application.is_prepared.name)
logger.debug('Found ShareDir directory: %s' % self.application.is_prepared.name)
elif self.application.is_prepared not in [None, True]:
shared_path = _getSharedPath()
if not os.path.isdir(os.path.join(shared_path, self.application.is_prepared.name)):
logger.error('ShareDir directory not found: %s' % self.application.is_prepared.name)
logger.error('Unpreparing Job #%s' % self.id)
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.application.is_prepared.name)
self.unprepare()
if unprepare is True:
if hasattr(self, 'is_prepared'):
from Ganga.Utility.files import expandfilename
if self.is_prepared not in [None, True]:
if hasattr(self.is_prepared, 'name'):
shared_path = _getSharedPath()
if not os.path.isdir(os.path.join(shared_path, self.is_prepared.name)):
logger.error('ShareDir directory not found: %s' % self.is_prepared.name)
logger.error('Unpreparing %s application' % getName(stripProxy(self)))
self.unprepare()
c = stripProxy(self).clone()
if hasattr(c, 'is_prepared') and c._getRegistry() is None:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.is_prepared.name)
stripProxy(c)._auto__init__(unprepare=True)
else:
c = stripProxy(self).clone()
stripProxy(c)._auto__init__()
return addProxy(c)
def test_all_exceptions(gpi):
"""Create all exceptions and make sure they behave correctly"""
import Ganga.Core.exceptions
test_str = "My Test Error"
def exception_test(err_name):
"""Run tests on the given exception"""
err_type = getattr(Ganga.Core.exceptions, err_name)
err_obj = err_type(test_str)
assert test_str in str(err_obj)
err_list = [
"GangaException", "GangaFileError", "PluginError",
"ApplicationConfigurationError", "ApplicationPrepareError",
"IncompleteJobSubmissionError", "IncompleteKillError",
"JobManagerError", "GangaAttributeError", "GangaValueError",
"GangaIOError", "SplitterError", "ProtectedAttributeError",
"ReadOnlyObjectError", "TypeMismatchError", "SchemaError",
"SchemaVersionError", "CredentialsError", "CredentialRenewalError",
"InvalidCredentialError", "ExpiredCredentialError"
]
for e in err_list:
exception_test(e)
# check the BackendError
from Ganga.Core.exceptions import BackendError
err = BackendError("TestBackend", test_str)
assert "TestBackend" in str(err)
assert test_str in str(err)
# check the InaccessibleObjectError
from Ganga.Core.exceptions import InaccessibleObjectError, JobManagerError
from Ganga.Core.GangaRepository import getRegistry
err = InaccessibleObjectError(
getRegistry('jobs').repository, 0,
JobManagerError("My JobManagerError"))
assert "jobs" in str(err)
assert "#0" in str(err)
assert "My JobManagerError" in str(err)
# check the RepositoryError
from Ganga.Core.exceptions import RepositoryError
from Ganga.Core.GangaRepository import getRegistry
RepositoryError(getRegistry('jobs').repository, test_str)
# Construct another to check the except clause in the exception is called
RepositoryError(getRegistry('jobs').repository, test_str)
def shutdown():
# Shutdown method for all repgistries in order
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.info('Registry Shutdown')
#import traceback
#traceback.print_stack()
# Flush all repos before we shut them down
flush_all()
# shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
# method to perform actions on the box and/or job registries.
logger.debug(started_registries)
all_registries = getRegistries()
try:
if 'prep' in started_registries:
registry = getRegistry('prep')
registry.shutdown()
# in case this is called repeatedly, only call shutdown once
started_registries.remove(registry.name)
except Exception as err:
logger.error("Err: %s" % err)
logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
logger.error("Trying to shutdown cleanly regardless")
for registry in getRegistries():
thisName = registry.name
try:
if not thisName in started_registries:
continue
# in case this is called repeatedly, only call shutdown once
started_registries.remove(thisName)
registry.shutdown() # flush and release locks
except Exception as x:
logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
logger.error("%s" % x)
logger.error("Trying to Shutdown cleanly regardless")
for registry in all_registries:
my_reg = [registry]
if hasattr(registry, 'metadata'):
if registry.metadata:
my_reg.append(registry.metadata)
assigned_attrs = ['location', 'type']
for this_reg in my_reg:
for attr in assigned_attrs:
if hasattr(registry, attr):
delattr(registry, attr)
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
removeRegistries()
def test_F_decrement(self):
"""Test that the ref counter is decremented as appropriate"""
from Ganga.GPI import jobs
j=jobs[-1]
shareRef = getRegistry('prep').getShareRef()
refDict = shareRef.name
assert refDict[j.application.is_prepared.name] is 2
this_ref = j.application.is_prepared.name
this_path = j.application.is_prepared.path()
j.unprepare()
assert refDict[this_ref] is 1
j2=jobs[-2].copy()
assert refDict[this_ref] is 2
j2.remove()
assert refDict[this_ref] is 1
jobs[-2].remove()
assert refDict[this_ref] is 0
assert not path.isdir(this_path)
def test_F_decrement(self):
"""Test that the ref counter is decremented as appropriate"""
from Ganga.GPI import jobs
j = jobs[-1]
shareRef = getRegistry('prep').getShareRef()
refDict = shareRef.name
assert refDict[j.application.is_prepared.name] is 2
this_ref = j.application.is_prepared.name
this_path = j.application.is_prepared.path()
j.unprepare()
assert refDict[this_ref] is 1
j2 = jobs[-2].copy()
assert refDict[this_ref] is 2
j2.remove()
assert refDict[this_ref] is 1
jobs[-2].remove()
assert refDict[this_ref] is 0
assert not path.isdir(this_path)
def __prep_set__(obj, val):
# if we set is_prepared to None in the GPI, that should effectively
# unprepare the application
if val is None:
if stripProxy(obj).is_prepared is not None:
logger.info('Unpreparing application.')
stripProxy(obj).unprepare()
# Replace is_prepared on an application for another ShareDir object
if hasattr(stripProxy(obj), '_getRegistry'):
from Ganga.GPIDev.Lib.File import ShareDir
if stripProxy(obj)._getRegistry() is not None and isType(
val, ShareDir):
logger.debug(
'Overwriting is_prepared attribute with a ShareDir object')
# it's safe to unprepare 'not-prepared' applications.
stripProxy(obj).unprepare()
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(
getRegistry("prep").getShareRef())
shareref.increase(val.name)
if type(val) is str:
logger.error("Setting string type to 'is_prepared'")
import traceback
traceback.print_stack()
def shutdown():
logger.debug("registry shutdown")
# shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
# method to perform actions on the box and/or job registries.
logger.debug(started_registries)
try:
if "prep" in started_registries:
registry = getRegistry("prep")
registry.shutdown()
# in case this is called repeatedly, only call shutdown once
started_registries.remove(registry.name)
except:
logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
logger.error("Trying to shutdown cleanly regardless")
pass
for registry in getRegistries():
thisName = registry.name
try:
if not thisName in started_registries:
continue
# in case this is called repeatedly, only call shutdown once
started_registries.remove(thisName)
registry.shutdown() # flush and release locks
except Exception as x:
logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
logger.error("%s" % str(x))
logger.error("Trying to Shutdown cleanly regardless")
pass
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
def __prep_set__(obj, val):
# if we set is_prepared to None in the GPI, that should effectively
# unprepare the application
if val is None:
if stripProxy(obj).is_prepared is not None:
logger.info("Unpreparing application.")
stripProxy(obj).unprepare()
# Replace is_prepared on an application for another ShareDir object
if hasattr(stripProxy(obj), "_getRegistry"):
from Ganga.GPIDev.Lib.File import ShareDir
if stripProxy(obj)._getRegistry() is not None and isType(val, ShareDir):
logger.debug("Overwriting is_prepared attribute with a ShareDir object")
# it's safe to unprepare 'not-prepared' applications.
stripProxy(obj).unprepare()
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(val.name)
if type(val) is str:
logger.error("Setting string type to 'is_prepared'")
import traceback
traceback.print_stack()
def __app_set__(obj, val):
if not hasattr(obj, 'application') or obj.application is None:
return
if hasattr(obj.application, '_is_prepared'):
#a=Job(); a.prepare(); a.application=Executable()
if obj.application.is_prepared not in [None, True] and\
hasattr(val, 'is_prepared') and val.is_prepared is None:
logger.debug('Overwriting a prepared application with one that is unprepared')
obj.application.unprepare()
#a=Job(); b=Executable(); b.prepare(); a.application=b
elif obj.application.is_prepared is not True:
if hasattr(val, 'is_prepared'):
if val.is_prepared not in [None, True]:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug('Overwriting application with a prepared one')
if stripProxy(obj.application) != val:
stripProxy(obj.application).unprepare()
shareref.increase(val.is_prepared.name)
# check that the shared directory actually exists before
# assigning the (prepared) application to a job
if hasattr(val, 'is_prepared'):
if val.is_prepared not in [None, True]:
if hasattr(val.is_prepared, 'name'):
from Ganga.Utility.files import expandfilename
Config_conf = getConfig('Configuration')
shared_path = os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user'])
if not os.path.isdir(os.path.join(shared_path, val.is_prepared.name)):
logger.error('ShareDir directory not found: %s' % val.is_prepared.name)
def run(self):
server, port = getHttpServer()
server.socket.settimeout(1)
reg = getRegistry("jobs")
"""
try:
server = HTTPServer(('pclcg35.cern.ch', 1234), GetHandler)
server.socket.settimeout(1)
except Exception:
return
print "Another Ganga session is already started with --webgui option"
process_details = getProcessDetails()
print reg.repository.get_other_sessions()
print "Process id : %s, hostname : %s" % (process_details[0], process_details[1])
self.stop()
self.unregister()
return
"""
print 'Starting web gui monitoring server, please wait ...'
# initialization
reg = getRegistry("jobs")
#calling here first time will take all jobs
reg.pollChangedJobs("WebGUI")
#fill jobs dictionary at the begining
fill_jobs_dictionary()
print 'Web gui monitoring server started successfully'
print
print 'You can monitor your jobs at the following location: ' + getMonitoringLink(port)
#server.serve_forever()
try:
while not self.should_stop():
server.handle_request()
finally:
pass
def test_0_testRegistryAsserts(self):
from Ganga.GPI import jobs, box, tasks, prep
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry
assert getRegistryProxy('jobs') is jobs
assert getRegistryProxy('box') is box
assert getRegistryProxy('tasks') is tasks
assert getRegistryProxy('prep') is prep
assert getRegistrySlice('jobs') is stripProxy(jobs)
assert getRegistrySlice('box') is stripProxy(box)
assert getRegistrySlice('tasks') is stripProxy(tasks)
assert getRegistrySlice('prep') is stripProxy(prep)
assert getRegistry('jobs') is stripProxy(jobs).objects
assert getRegistry('tasks') is stripProxy(tasks).objects
assert getRegistry('box') is stripProxy(box).objects
def listShareDirContents(self, shared_directory_name):
"""
Function which is used to list the contents of the prepared sandbox folder managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.ls(shared_directory_name)
def test_0_testRegistryAsserts(self):
from Ganga.GPI import jobs, box, tasks, prep
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry
assert getRegistryProxy('jobs') is jobs
assert getRegistryProxy('box') is box
assert getRegistryProxy('tasks') is tasks
assert getRegistryProxy('prep') is prep
assert getRegistrySlice('jobs') is stripProxy(jobs)
assert getRegistrySlice('box') is stripProxy(box)
assert getRegistrySlice('tasks') is stripProxy(tasks)
assert getRegistrySlice('prep') is stripProxy(prep)
assert getRegistry('jobs') is stripProxy(jobs).objects
assert getRegistry('tasks') is stripProxy(tasks).objects
assert getRegistry('box') is stripProxy(box).objects
def listShareDirContents(self, shared_directory_name):
"""
Function which is used to list the contents of the prepared sandbox folder managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.ls(shared_directory_name)
def run(self):
server, port = getHttpServer()
server.socket.settimeout(1)
reg = getRegistry("jobs")
"""
try:
server = HTTPServer(('pclcg35.cern.ch', 1234), GetHandler)
server.socket.settimeout(1)
except Exception:
return
print("Another Ganga session is already started with --webgui option" )
process_details = getProcessDetails()
print(reg.repository.get_other_sessions())
print("Process id : %s, hostname : %s" % (process_details[0], process_details[1]))
self.stop()
self.unregister()
return
"""
logger.info('Starting web gui monitoring server, please wait ...')
# initialization
reg = getRegistry("jobs")
# calling here first time will take all jobs
reg.pollChangedJobs("WebGUI")
# fill jobs dictionary at the begining
fill_jobs_dictionary()
logger.info('Web gui monitoring server started successfully')
logger.info('You can monitor your jobs at the following location: ' + getMonitoringLink(port))
# server.serve_forever()
try:
while not self.should_stop():
server.handle_request()
finally:
pass
def _auto__init__(self, registry=None):
if registry is None:
from Ganga.Core.GangaRepository import getRegistry
registry = getRegistry(self.default_registry)
# register the job (it will also commit it)
# job gets its id now
registry._add(self)
self.creation_date = time.strftime('%Y%m%d%H%M%S')
self.startup()
self.status = 'new'
def _auto__init__(self, registry=None):
if registry is None:
from Ganga.Core.GangaRepository import getRegistry
registry = getRegistry(self.default_registry)
# register the job (it will also commit it)
# job gets its id now
registry._add(self)
self.creation_date = time.strftime('%Y%m%d%H%M%S')
self.startup()
self.status = 'new'
def _copy(self, unprepare=None):
logger.debug('unprepare is %s', str(unprepare))
if unprepare is None:
if prepconfig['unprepare_on_copy'] is True:
if hasattr(self,'is_prepared') or hasattr(self,'application'):
unprepare = True
if hasattr(self,'application'):
if hasattr(self.application,'is_prepared'):
if self.application.is_prepared is not None and self.application.is_prepared \
is not True and os.path.isdir(os.path.join(shared_path,self.application.is_prepared.name)):
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug('increasing counter from proxy.py')
shareref.increase(self.application.is_prepared.name)
logger.debug('Found ShareDir directory: %s' % self.application.is_prepared.name)
elif self.application.is_prepared is not None and self.application.is_prepared \
is not True and not os.path.isdir(os.path.join(shared_path,self.application.is_prepared.name)):
logger.error('ShareDir directory not found: %s' % self.application.is_prepared.name)
logger.error('Unpreparing Job #%s' % self.id)
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.application.is_prepared.name)
self.unprepare()
if hasattr(self,'is_prepared') and self.is_prepared is not None and self.is_prepared \
is not True and not os.path.isdir(os.path.join(shared_path,self.is_prepared.name)):
logger.error('ShareDir directory not found: %s' % self.is_prepared.name)
logger.error('Unpreparing %s application' % self._impl._name)
self.unprepare()
if unprepare is True:
c = self._impl.clone()
if hasattr(c,'is_prepared') and c._getRegistry() is None:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.is_prepared.name)
c._auto__init__(unprepare=True)
else:
c = self._impl.clone()
c._auto__init__()
return GPIProxyObjectFactory(c)
def test_all_exceptions(gpi):
"""Create all exceptions and make sure they behave correctly"""
import Ganga.Core.exceptions
test_str = "My Test Error"
def exception_test(err_name):
"""Run tests on the given exception"""
err_type = getattr(Ganga.Core.exceptions, err_name)
err_obj = err_type(test_str)
assert test_str in str(err_obj)
err_list = ["GangaException", "GangaFileError", "PluginError", "ApplicationConfigurationError",
"ApplicationPrepareError", "IncompleteJobSubmissionError", "IncompleteKillError", "JobManagerError",
"GangaAttributeError", "GangaValueError", "GangaIOError", "SplitterError", "ProtectedAttributeError",
"ReadOnlyObjectError", "TypeMismatchError", "SchemaError", "SchemaVersionError", "CredentialsError",
"CredentialRenewalError", "InvalidCredentialError", "ExpiredCredentialError"]
for e in err_list:
exception_test(e)
# check the BackendError
from Ganga.Core.exceptions import BackendError
err = BackendError("TestBackend", test_str)
assert "TestBackend" in str(err)
assert test_str in str(err)
# check the InaccessibleObjectError
from Ganga.Core.exceptions import InaccessibleObjectError, JobManagerError
from Ganga.Core.GangaRepository import getRegistry
err = InaccessibleObjectError(getRegistry('jobs').repository, 0, JobManagerError("My JobManagerError"))
assert "jobs" in str(err)
assert "#0" in str(err)
assert "My JobManagerError" in str(err)
# check the RepositoryError
from Ganga.Core.exceptions import RepositoryError
from Ganga.Core.GangaRepository import getRegistry
RepositoryError(getRegistry('jobs').repository, test_str)
# Construct another to check the except clause in the exception is called
RepositoryError(getRegistry('jobs').repository, test_str)
def __incrementShareRef(obj, attr_name):
shared_dir = getattr(obj, attr_name)
if hasattr(shared_dir, 'name'):
from Ganga.Core.GangaRepository import getRegistry
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug("Increasing shareref")
shareref.increase(shared_dir.name)
def decrementShareCounter(self, shared_directory_name, remove=0):
"""
Function which is used to decrement the number of (sub)jobs which share the prepared sandbox
managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
remove = remove
logger.debug('Decrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.decrease(shared_directory_name, remove)
def incrementShareCounter(self, shared_directory_name):
"""
Function which is used to increment the number of (sub)jobs which share the prepared sandbox
managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
logger.debug('Incrementing shared directory reference counter')
shareref = getRegistry("prep").getShareRef()
logger.debug('within incrementShareCounter, calling increase')
shareref.increase(shared_directory_name)
def decrementShareCounter(self, shared_directory_name, remove=0):
"""
Function which is used to decrement the number of (sub)jobs which share the prepared sandbox
managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
remove = remove
logger.debug('Decrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.decrease(shared_directory_name, remove)
def incrementShareCounter(self, shared_directory_name):
"""
Function which is used to increment the number of (sub)jobs which share the prepared sandbox
managed by this app
Args:
shared_directory_name (str): full name of directory managed by this app
"""
logger.debug('Incrementing shared directory reference counter')
shareref = getRegistry("prep").getShareRef()
logger.debug('within incrementShareCounter, calling increase')
shareref.increase(shared_directory_name)
def add(self, input):
if not isType(input, list):
input = [input]
for item in input:
if isType(item, str):
if os.path.isfile(expandfilename(item)):
logger.info('Copying file %s to shared directory %s'%(item, self.name))
shutil.copy2(expandfilename(item), os.path.join(shared_path,self.name))
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item))
elif isType(item,File) and item.name is not '' and os.path.isfile(expandfilename(item.name)):
logger.info('Copying file object %s to shared directory %s'%(item.name,self.name))
shutil.copy2(expandfilename(item.name), os.path.join(shared_path,self.name))
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item.name))
def add(self, input):
from Ganga.Core.GangaRepository import getRegistry
if not isType(input, list):
input = [input]
for item in input:
if isType(item, str):
if os.path.isfile(expandfilename(item)):
logger.info('Copying file %s to shared directory %s' % (item, self.name))
shutil.copy2(expandfilename(item), os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item))
elif isType(item, File) and item.name is not '' and os.path.isfile(expandfilename(item.name)):
logger.info('Copying file object %s to shared directory %s' % (item.name, self.name))
shutil.copy2(expandfilename(item.name), os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item.name))
def shutdown():
logger.debug('registry shutdown')
#shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
#method to perform actions on the box and/or job registries.
logger.debug(started_registries)
if 'prep' in started_registries:
registry = getRegistry('prep')
registry.shutdown()
started_registries.remove(registry.name) # in case this is called repeatedly, only call shutdown once
for registry in getRegistries():
if not registry.name in started_registries: continue
started_registries.remove(registry.name) # in case this is called repeatedly, only call shutdown once
registry.shutdown() # flush and release locks
def test_E_increment(self):
"""Test that the ref counter is incremented as appropriate"""
from Ganga.GPI import jobs
j=jobs[-1]
shareRef = getRegistry('prep').getShareRef()
refDict = shareRef.name
assert j.application.is_prepared.name in refDict
j2 = j.copy()
assert refDict[j.application.is_prepared.name] is 2
def test_E_increment(self):
"""Test that the ref counter is incremented as appropriate"""
from Ganga.GPI import jobs
j = jobs[-1]
shareRef = getRegistry('prep').getShareRef()
refDict = shareRef.name
assert j.application.is_prepared.name in refDict
j2 = j.copy()
assert refDict[j.application.is_prepared.name] is 2
def __deepcopy__(self, memo = None):
self._getReadAccess()
c = super(GangaObject,self).__deepcopy__(memo)
for name,item in self._schema.allItems():
if not item['copyable']:
setattr(c,name,self._schema.getDefaultValue(name))
if item.isA(Schema.SharedItem):
shared_dir=getattr(c,name)
try:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug("Increasing shareref")
shareref.increase(shared_dir.name)
except AttributeError:
pass
return c
def update_jobs_dictionary():
reg = getRegistry("jobs")
#get the changed jobs
changed_ids = reg.pollChangedJobs("WebGUI")
for job_id in changed_ids:
try:
job = jobs(job_id)
try:
jobs_dictionary[job_id] = JobRelatedInfo(job, job.time.timestamps['new'])
except RegistryKeyError:
jobs_dictionary[job_id] = JobRelatedInfo(job, None)
except RegistryKeyError:
del jobs_dictionary[job_id]
def update_jobs_dictionary():
from Ganga.Core.GangaRepository import getRegistryProxy
reg = getRegistry("jobs")
# get the changed jobs
changed_ids = reg.pollChangedJobs("WebGUI")
for job_id in changed_ids:
try:
job = getRegistryProxy('jobs')(job_id)
try:
jobs_dictionary[job_id] = JobRelatedInfo(
job, job.time.timestamps['new'])
except RegistryKeyError:
jobs_dictionary[job_id] = JobRelatedInfo(job, None)
except RegistryKeyError:
del jobs_dictionary[job_id]
def __app_set__(obj, val):
if hasattr(obj.application, '_is_prepared'):
#a=Job(); a.prepare(); a.application=Executable()
if obj.application.is_prepared not in [None, True] and\
hasattr(val, 'is_prepared') and val.is_prepared is None:
logger.debug(
'Overwriting a prepared application with one that is unprepared'
)
obj.application.unprepare()
#a=Job(); b=Executable(); b.prepare(); a.application=b
elif obj.application.is_prepared is not True:
if hasattr(val, 'is_prepared'):
if val.is_prepared not in [None, True]:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(
getRegistry("prep").getShareRef())
logger.debug('Overwriting application with a prepared one')
if stripProxy(obj.application) != val:
stripProxy(obj.application).unprepare()
shareref.increase(val.is_prepared.name)
# check that the shared directory actually exists before
# assigning the (prepared) application to a job
if hasattr(val, 'is_prepared'):
if val.is_prepared not in [None, True]:
if hasattr(val.is_prepared, 'name'):
from Ganga.Utility.files import expandfilename
Config_conf = getConfig('Configuration')
shared_path = os.path.join(
expandfilename(Config_conf['gangadir']), 'shared',
Config_conf['user'])
if not os.path.isdir(
os.path.join(shared_path,
val.is_prepared.name)):
logger.error('ShareDir directory not found: %s' %
val.is_prepared.name)
def __deepcopy__(self, memo=None):
self = stripProxy(self)
self._getReadAccess()
c = super(GangaObject, self).__deepcopy__(memo)
if self._schema is not None:
for name, item in self._schema.allItems():
if not item['copyable']:
setattr(c, name, self._schema.getDefaultValue(name))
if item.isA(Schema.SharedItem):
shared_dir = getattr(c, name)
if hasattr(shared_dir, 'name'):
from Ganga.Core.GangaRepository import getRegistry
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug("Increasing shareref")
shareref.increase(shared_dir.name)
return c
def shutdown():
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.info('Registry Shutdown')
#import traceback
#traceback.print_stack()
# shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
# method to perform actions on the box and/or job registries.
logger.debug(started_registries)
try:
if 'prep' in started_registries:
registry = getRegistry('prep')
registry.shutdown()
# in case this is called repeatedly, only call shutdown once
started_registries.remove(registry.name)
except Exception as err:
logger.debug("Err: %s" % str(err))
logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
logger.error("Trying to shutdown cleanly regardless")
for registry in getRegistries():
thisName = registry.name
try:
if not thisName in started_registries:
continue
# in case this is called repeatedly, only call shutdown once
started_registries.remove(thisName)
registry.shutdown() # flush and release locks
except Exception as x:
logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
logger.error("%s" % str(x))
logger.error("Trying to Shutdown cleanly regardless")
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
def shutdown():
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.info('Registry Shutdown')
#import traceback
#traceback.print_stack()
# shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
# method to perform actions on the box and/or job registries.
logger.debug(started_registries)
try:
if 'prep' in started_registries:
registry = getRegistry('prep')
registry.shutdown()
# in case this is called repeatedly, only call shutdown once
started_registries.remove(registry.name)
except Exception as err:
logger.debug("Err: %s" % str(err))
logger.error("Failed to Shutdown prep Repository!!! please check for stale lock files")
logger.error("Trying to shutdown cleanly regardless")
for registry in getRegistries():
thisName = registry.name
try:
if not thisName in started_registries:
continue
# in case this is called repeatedly, only call shutdown once
started_registries.remove(thisName)
registry.shutdown() # flush and release locks
except Exception as x:
logger.error("Failed to Shutdown Repository: %s !!! please check for stale lock files" % thisName)
logger.error("%s" % str(x))
logger.error("Trying to Shutdown cleanly regardless")
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
def listShareDirContents(self, shared_directory_name):
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.ls(shared_directory_name)
def listShareDirs(self):
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref
def _thread_main(self):
""" This is an internal function; the main loop of the background thread """
from Ganga.Core.GangaRepository import getRegistry
while getRegistry("jobs").hasStarted() is not True:
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
while True:
from Ganga.Core import monitoring_component
if (not monitoring_component is None and monitoring_component.enabled) or config['ForceTaskMonitoring']:
break
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
# setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY
# and must go away soon
for tid in self.ids():
try:
self[tid].startup()
except Exception as err:
logger.error("Unknown/Unexpected Error in starting up tasks main loop")
logger.error("Exiting: err=%s" % str(err))
return
logger.debug("Entering main loop")
# Main loop
while self._main_thread is not None and not self._main_thread.should_stop():
# If monitoring is enabled (or forced for Tasks) loop over each one and update
if (config['ForceTaskMonitoring'] or monitoring_component.enabled) and not config['disableTaskMon']:
for tid in self.ids():
logger.debug("Running over tid: %s" % str(tid))
try:
p = self[tid]
p.update()
except Exception as x:
logger.error(
"Exception occurred in task monitoring loop: %s %s\nThe offending task was paused." % (x.__class__, x))
type_, value_, traceback_ = sys.exc_info()
logger.error("Full traceback:\n %s" % ' '.join(
traceback.format_exception(type_, value_, traceback_)))
p.pause()
if self._main_thread.should_stop():
break
if self._main_thread.should_stop():
break
logger.debug("TaskRegistry Sleeping for: %s seconds" % str(config['TaskLoopFrequency']))
# Sleep interruptible for 10 seconds
for i in range(0, int(config['TaskLoopFrequency'] * 100)):
if self._main_thread.should_stop():
break
time.sleep(0.01)
def decrementShareCounter(self, shared_directory_name, remove=0):
remove = remove
logger.debug('Decrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.decrease(shared_directory_name, remove)
def listShareDirs(self):
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref
def listShareDirContents(self,shared_directory_name):
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.ls(shared_directory_name)
def shutdown():
# Shutdown method for all repgistries in order
from Ganga.Utility.logging import getLogger
logger = getLogger()
logger.info('Registry Shutdown')
#import traceback
#traceback.print_stack()
# Flush all repos before we shut them down
flush_all()
# shutting down the prep registry (i.e. shareref table) first is necessary to allow the closedown()
# method to perform actions on the box and/or job registries.
logger.debug(started_registries)
all_registries = getRegistries()
try:
if 'prep' in started_registries:
registry = getRegistry('prep')
registry.shutdown()
# in case this is called repeatedly, only call shutdown once
started_registries.remove(registry.name)
except Exception as err:
logger.error("Err: %s" % err)
logger.error(
"Failed to Shutdown prep Repository!!! please check for stale lock files"
)
logger.error("Trying to shutdown cleanly regardless")
for registry in getRegistries():
thisName = registry.name
try:
if not thisName in started_registries:
continue
# in case this is called repeatedly, only call shutdown once
started_registries.remove(thisName)
registry.shutdown() # flush and release locks
except Exception as x:
logger.error(
"Failed to Shutdown Repository: %s !!! please check for stale lock files"
% thisName)
logger.error("%s" % x)
logger.error("Trying to Shutdown cleanly regardless")
for registry in all_registries:
my_reg = [registry]
if hasattr(registry, 'metadata'):
if registry.metadata:
my_reg.append(registry.metadata)
assigned_attrs = ['location', 'type']
for this_reg in my_reg:
for attr in assigned_attrs:
if hasattr(registry, attr):
delattr(registry, attr)
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
removeRegistries()
def decrementShareCounter(self, shared_directory_name, remove=0):
remove=remove
logger.debug('Decrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.decrease(shared_directory_name, remove)
convert_merger_to_postprocessor, 'Functions')
exportToPublicInterface('config', Ganga.GPIDev.Lib.Config.config, 'Objects',
'access to Ganga configuration')
exportToPublicInterface('ConfigError', Ganga.GPIDev.Lib.Config.ConfigError,
'Exceptions')
exportToPublicInterface('report', report, 'Functions')
# ------------------------------------------------------------------------------------
# bootstrap the repositories and connect to them
for n, k, d in Repository_runtime.bootstrap():
# make all repository proxies visible in GPI
exportToPublicInterface(n, k, 'Objects', d)
# ------------------------------------------------------------------------------------
# JobTree
jobtree = GPIProxyObjectFactory(getRegistry("jobs").getJobTree())
exportToPublicInterface('jobtree', jobtree, 'Objects',
'Logical tree view of the jobs')
exportToPublicInterface('TreeError', TreeError, 'Exceptions')
# ------------------------------------------------------------------------------------
# ShareRef
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
exportToPublicInterface(
'shareref', shareref, 'Objects',
'Mechanism for tracking use of shared directory resources')
# ------------------------------------------------------------------------------------
# bootstrap the workspace
Workspace_runtime.bootstrap()
def incrementShareCounter(self, shared_directory_name):
logger.debug('Incrementing shared directory reference counter')
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug('within incrementShareCounter, calling increase')
shareref.increase(shared_directory_name)
def _thread_main(self):
""" This is an internal function; the main loop of the background thread """
from Ganga.Core.GangaRepository import getRegistry
while getRegistry("jobs").hasStarted() is not True:
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
while True:
from Ganga.Core import monitoring_component
if (not monitoring_component is None
and monitoring_component.enabled
) or config['ForceTaskMonitoring']:
break
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
# setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY
# and must go away soon
for tid in self.ids():
try:
self[tid].startup()
except Exception as err:
logger.error(
"Unknown/Unexpected Error in starting up tasks main loop")
logger.error("Exiting: err=%s" % str(err))
return
logger.debug("Entering main loop")
# Main loop
while self._main_thread is not None and not self._main_thread.should_stop(
):
# If monitoring is enabled (or forced for Tasks) loop over each one and update
if (config['ForceTaskMonitoring'] or monitoring_component.enabled
) and not config['disableTaskMon']:
for tid in self.ids():
logger.debug("Running over tid: %s" % str(tid))
try:
p = self[tid]
p.update()
except Exception as x:
logger.error(
"Exception occurred in task monitoring loop: %s %s\nThe offending task was paused."
% (x.__class__, x))
type_, value_, traceback_ = sys.exc_info()
logger.error("Full traceback:\n %s" % ' '.join(
traceback.format_exception(type_, value_,
traceback_)))
p.pause()
if self._main_thread.should_stop():
break
if self._main_thread.should_stop():
break
logger.debug("TaskRegistry Sleeping for: %s seconds" %
str(config['TaskLoopFrequency']))
# Sleep interruptible for 10 seconds
for i in range(0, int(config['TaskLoopFrequency'] * 100)):
if self._main_thread.should_stop():
break
time.sleep(0.01)
def _copy(self, unprepare=None):
logger.debug('unprepare is %s', str(unprepare))
if unprepare is None:
if prepconfig['unprepare_on_copy'] is True:
if hasattr(self, 'is_prepared') or hasattr(
self, 'application'):
unprepare = True
def _getSharedPath():
Config_conf = getConfig('Configuration')
return os.path.join(expandfilename(Config_conf['gangadir']),
'shared', Config_conf['user'])
if hasattr(self, 'application'):
if hasattr(self.application, 'is_prepared'):
from Ganga.Utility.files import expandfilename
if self.application.is_prepared not in [None, True]:
if hasattr(self.application.is_prepared, 'name'):
shared_path = _getSharedPath()
if os.path.isdir(
os.path.join(
shared_path,
self.application.is_prepared.name)):
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(
getRegistry("prep").getShareRef())
logger.debug('increasing counter from proxy.py')
shareref.increase(
self.application.is_prepared.name)
logger.debug('Found ShareDir directory: %s' %
self.application.is_prepared.name)
elif self.application.is_prepared not in [None, True]:
shared_path = _getSharedPath()
if not os.path.isdir(
os.path.join(shared_path,
self.application.is_prepared.name)):
logger.error('ShareDir directory not found: %s' %
self.application.is_prepared.name)
logger.error('Unpreparing Job #%s' % self.id)
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(
getRegistry("prep").getShareRef())
shareref.increase(self.application.is_prepared.name)
self.unprepare()
if unprepare is True:
if hasattr(self, 'is_prepared'):
from Ganga.Utility.files import expandfilename
if self.is_prepared not in [None, True]:
if hasattr(self.is_prepared, 'name'):
shared_path = _getSharedPath()
if not os.path.isdir(
os.path.join(shared_path,
self.is_prepared.name)):
logger.error('ShareDir directory not found: %s' %
self.is_prepared.name)
logger.error('Unpreparing %s application' %
getName(stripProxy(self)))
self.unprepare()
c = stripProxy(self).clone()
if hasattr(c, 'is_prepared') and c._getRegistry() is None:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(
getRegistry("prep").getShareRef())
shareref.increase(self.is_prepared.name)
stripProxy(c)._auto__init__(unprepare=True)
else:
c = stripProxy(self).clone()
stripProxy(c)._auto__init__()
return GPIProxyObjectFactory(c)
def _thread_main(self):
""" This is an internal function; the main loop of the background thread """
# Add runtime handlers for all the taskified applications, since now
# all the backends are loaded
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from .TaskApplication import handler_map
for basename, name in handler_map:
for backend in allHandlers.getAllBackends(basename):
allHandlers.add(name, backend,
allHandlers.get(basename, backend))
from Ganga.Core.GangaRepository import getRegistry
while getRegistry("jobs").hasStarted() is not True:
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
while True:
from Ganga.Core import monitoring_component
if (not monitoring_component is None
and monitoring_component.enabled
) or config['ForceTaskMonitoring']:
break
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
# setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY
# and must go away soon
for tid in self.ids():
try:
self[tid]._getWriteAccess()
self[tid].startup()
except RegistryError:
continue
except Exception as err:
logger.error(
"Unknown/Unexpected Error in starting up tasks main loop")
logger.error("Exiting: err=%s" % str(err))
return
logger.debug("Entering main loop")
# Main loop
while self._main_thread is not None and not self._main_thread.should_stop(
):
# For each task try to run it
if config['ForceTaskMonitoring'] or monitoring_component.enabled:
for tid in self.ids():
logger.debug("Running over tid: %s" % str(tid))
try:
from Ganga.GPIDev.Lib.Tasks import ITask
if isType(self[tid], ITask):
# for new ITasks, always need write access
self[tid]._getWriteAccess()
p = self[tid]
else:
if self[tid].status in [
"running", "running/pause"
]:
self[tid]._getWriteAccess()
p = self[tid]
elif self[tid].status is 'completed' and (
self[tid].n_status('ready')
or self[tid].n_status('running')):
self[tid].updateStatus()
continue
else:
continue
except RegistryError:
# could not acquire lock
continue
if self._main_thread.should_stop():
break
try:
from Ganga.GPIDev.Lib.Tasks import ITask
if isType(self[tid], ITask):
# for new ITasks, always call update()
p.update()
else:
# TODO: Make this user-configurable and add better
# error message
if (p.n_status("failed") * 100.0 /
(20 + p.n_status("completed")) > 20):
p.pause()
logger.error(
"Task %s paused - %i jobs have failed while only %i jobs have completed successfully."
% (p.name, p.n_status("failed"),
p.n_status("completed")))
logger.error(
"Please investigate the cause of the failing jobs and then remove the previously failed jobs using job.remove()"
)
logger.error(
"You can then continue to run this task with tasks(%i).run()"
% p.id)
continue
numjobs = p.submitJobs()
if numjobs > 0:
self._flush([p])
# finalise any required transforms
p.finaliseTransforms()
p.updateStatus()
except Exception as x:
logger.error(
"Exception occurred in task monitoring loop: %s %s\nThe offending task was paused."
% (x.__class__, x))
type_, value_, traceback_ = sys.exc_info()
logger.error("Full traceback:\n %s" % ' '.join(
traceback.format_exception(type_, value_,
traceback_)))
p.pause()
if self._main_thread.should_stop():
break
if self._main_thread.should_stop():
break
logger.debug("TaskRegistry Sleeping for: %s seconds" %
str(config['TaskLoopFrequency']))
# Sleep interruptible for 10 seconds
for i in range(0, int(config['TaskLoopFrequency'] * 100)):
if self._main_thread.should_stop():
break
time.sleep(0.01)