def setUp(self):
super(Savannah9638, self).setUp()
wipe_temp_dir()
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'gangadir', '/tmp/ganga_topdir-$USER')
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
super(TestLazyLoadingSubjobs, self).setUp()
from Ganga.Utility.Config import getConfig
default_CleanUp = getConfig('TestingFramework')['AutoCleanup']
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def setUp(self):
"""Make sure that the Tasks object isn't destroyed between tests"""
super(TestCoreTasks, self).setUp()
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
setConfigOption('Tasks', 'TaskLoopFrequency', 1)
self._numTasks = 50
def test_e_testInMemory(self):
"""
Test the resubmit on a job in memory vs a job which has been loaded from disk
"""
from Ganga.GPI import Job, Local
j=Job()
j.splitter = self._getSplitter()
j.backend = Local()
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
# Job has ben created, split, run and now exists in Memory (NOT SJXML)
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'True')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).resubmit()
# We should get here if calling resubmit doesn't stall
j.subjobs(0).force_status('failed')
j.resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test resubmit from the master job worked
j.subjobs(0).force_status('failed')
j.subjobs(0).resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test that the resubmit from the subjob worked
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'False')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).force_status('failed')
j.resubmit()
sleep_until_completed(j)
def test_e_testInMemory(self):
"""
Test the resubmit on a job in memory vs a job which has been loaded from disk
"""
from Ganga.GPI import Job, Local
j = Job()
j.splitter = self._getSplitter()
j.backend = Local()
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
# Job has ben created, split, run and now exists in Memory (NOT SJXML)
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'True')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).resubmit()
# We should get here if calling resubmit doesn't stall
j.subjobs(0).force_status('failed', force=True)
j.resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test resubmit from the master job worked
j.subjobs(0).force_status('failed')
j.subjobs(0).resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test that the resubmit from the subjob worked
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'False')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).force_status('failed')
j.resubmit()
sleep_until_completed(j)
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
super(TestMutableMethods, self).setUp()
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
from Ganga.GPI import Job, TestSubmitter, GListApp
self.test_job = Job(application=GListApp(gListComp=[self._makeRandomTFile() for _ in range(10)],
gList=[self._makeRandomString() for _ in range(10)],
seq=range(10)),backend=TestSubmitter())
def test_d_JobRemoval(self):
""" Fourth make sure that we get rid of the jobs safely"""
from Ganga.GPI import jobs
self.assertEqual(len(jobs), 1)
jobs(0).remove()
self.assertEqual(len(jobs), 0)
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', default_CleanUp)
def test_d_JobRemoval(self):
""" Fourth make sure that we get rid of the jobs safely"""
from Ganga.GPI import jobs
self.assertEqual(len(jobs), 1)
jobs(0).remove()
self.assertEqual(len(jobs), 0)
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'True')
def test_CondorConfigDefaults(self):
from Ganga.GPI import Condor, GangaList
from Ganga.Utility.Config import setConfigOption
from Ganga.GPIDev.Base.Proxy import isType
setConfigOption('defaults_CondorRequirements', 'other', ['POOL == "General"'])
setConfigOption('defaults_CondorRequirements', 'opsys', 'print')
a = Condor()
c = a.requirements
assert isType(c.other, GangaList)
assert c.opsys == 'print'
assert c.other == ['POOL == "General"']
def test_CondorConfigDefaults(self):
from Ganga.GPI import Condor, GangaList
from Ganga.Utility.Config import setConfigOption
from Ganga.GPIDev.Base.Proxy import isType
setConfigOption('defaults_CondorRequirements', 'other',
['POOL == "General"'])
setConfigOption('defaults_CondorRequirements', 'opsys', 'print')
a = Condor()
c = a.requirements
assert isType(c.other, GangaList)
assert c.opsys == 'print'
assert c.other == ['POOL == "General"']
def test_unprepareTrue(self):
from Ganga.Utility.Config import setConfigOption
from Ganga.GPI import Job, Executable
setConfigOption('Preparable', 'unprepare_on_copy', 'True')
j = Job(application=Executable(exe='/bin/echo', args=['hello']))
j.submit()
assert (j.application.is_prepared != None)
j2 = j.copy()
assert (j2.application.is_prepared == None)
j3 = Job(j)
assert (j3.application.is_prepared == None)
def test_unprepareFalse(self):
from Ganga.Utility.Config import setConfigOption
from Ganga.GPI import Job, Executable
setConfigOption('Preparable', 'unprepare_on_copy', 'False')
k = Job(application=Executable(exe='/bin/echo', args=['hello']))
k.submit()
assert(k.application.is_prepared != None)
k2 = k.copy()
assert(k2.application.is_prepared != None)
k3 = Job(k)
assert(k.application.is_prepared != None)
def test_unprepareFalse(self):
from Ganga.Utility.Config import setConfigOption
from Ganga.GPI import Job, Executable
setConfigOption('Preparable', 'unprepare_on_copy', 'False')
k = Job(application=Executable(exe='/bin/echo', args=['hello']))
k.submit()
assert (k.application.is_prepared != None)
k2 = k.copy()
assert (k2.application.is_prepared != None)
k3 = Job(k)
assert (k.application.is_prepared != None)
def test_unprepareTrue(self):
from Ganga.Utility.Config import setConfigOption
from Ganga.GPI import Job, Executable
setConfigOption('Preparable', 'unprepare_on_copy', 'True')
j = Job(application=Executable(exe='/bin/echo', args=['hello']))
j.submit()
assert(j.application.is_prepared != None)
j2 = j.copy()
assert(j2.application.is_prepared == None)
j3 = Job(j)
assert(j3.application.is_prepared == None)
def bootstrap():
# Bootstrap for startup and setting of parameters for the Registries
retval = []
try:
checkDiskQuota()
except GangaException as err:
raise
except Exception as err:
logger.error("Disk quota check failed due to: %s" % err)
for registry in bootstrap_getreg():
if registry.name in started_registries:
continue
if not hasattr(registry, 'type'):
registry.type = config["repositorytype"]
if not hasattr(registry, 'location'):
registry.location = getLocalRoot()
logger.debug("Registry: %s" % registry.name)
logger.debug("Loc: %s" % registry.location)
registry.startup()
logger.debug("started " + registry.info(full=False))
started_registries.append(registry.name)
proxied_registry_slice = registry.getProxy()
retval.append((registry.name, proxied_registry_slice, registry.doc))
# Assuming all registries are started for all instances of Ganga atm
# Avoid ever, ever, calling the repository from behind the registry. This allows for all forms of bad behaviour
# This is a form of trade off but still is something which should be strongly discouraged!
# TODO investigate putting access to the repository behind a getter/setter which keeps the registry locked
other_sessions = bootstrap_getreg()[0].repository.get_other_sessions()
if other_sessions:
# Just print this from 1 repo only so chose the zeorth, nothing special
logger.warning("%i other concurrent sessions:\n * %s" %
(len(other_sessions), "\n * ".join(other_sessions)))
logger.warning(
"Multiple Ganga sessions detected. The Monitoring Thread is being disabled."
)
logger.warning("Type 'enableMonitoring' to restart")
setConfigOption('PollThread', 'autostart', False)
return retval
def _ganga_run_exitfuncs():
"""run any registered exit functions
atexit._exithandlers is traversed based on the priority.
If no priority was registered for a given function
than the lowest priority is assumed (LIFO policy)
We keep the same functionality as in *atexit* bare module but
we run each exit handler inside a try..catch block to be sure all
the registered handlers are executed
"""
# Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'DiskIOTimeout', 1)
try:
from Ganga.GPI import queues
queues.lock()
except Exception, err:
logger.debug("This should only happen if Ganga filed to initialize correctly")
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
super(TestSJSubmit, self).setUp()
from Ganga.Utility.Config import setConfigOption
setConfigOption('Output', 'FailJobIfNoOutputMatched', 'True')
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'True')
def bootstrap():
# Bootstrap for startup and setting of parameters for the Registries
retval = []
try:
checkDiskQuota()
except GangaException as err:
raise
except Exception as err:
logger.error("Disk quota check failed due to: %s" % err)
for registry in bootstrap_getreg():
if registry.name in started_registries:
continue
if not hasattr(registry, 'type'):
registry.type = config["repositorytype"]
if not hasattr(registry, 'location'):
registry.location = getLocalRoot()
logger.debug("Registry: %s" % registry.name)
logger.debug("Loc: %s" % registry.location)
registry.startup()
logger.debug("started " + registry.info(full=False))
started_registries.append(registry.name)
proxied_registry_slice = registry.getProxy()
retval.append((registry.name, proxied_registry_slice, registry.doc))
# Assuming all registries are started for all instances of Ganga atm
# Avoid ever, ever, calling the repository from behind the registry. This allows for all forms of bad behaviour
# This is a form of trade off but still is something which should be strongly discouraged!
# TODO investigate putting access to the repository behind a getter/setter which keeps the registry locked
other_sessions = bootstrap_getreg()[0].repository.get_other_sessions()
if other_sessions:
# Just print this from 1 repo only so chose the zeorth, nothing special
logger.warning("%i other concurrent sessions:\n * %s" % (len(other_sessions), "\n * ".join(other_sessions)))
logger.warning("Multiple Ganga sessions detected. The Monitoring Thread is being disabled.")
logger.warning("Type 'enableMonitoring' to restart")
setConfigOption('PollThread', 'autostart', False)
return retval
def test_d_SJResubmit_FailNotRequired(self):
"""
Resubmit when jobs are not required to have failed
"""
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'False')
from Ganga.GPI import jobs
# Test that resubmitting a job with SubJobXMLList subjobs doesn't stall
# Test them with all subjobs completed and resubmitOnlyFailedSubjobs = False
jobs(0).resubmit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(jobs(0))
# Test that resubmitting a subjob from SubJobXMLList that subjob doesn't stall
jobs(0).subjobs(0).resubmit()
jobs(0).subjobs(0).force_status('failed', force=True)
jobs(0).subjobs(0).resubmit()
sleep_until_completed(jobs(0))
assert jobs(0).subjobs(0).status == 'completed'
# Test that we can resubmit 1 subjob from the subjob itself when the subjob is failed
# resubmitOnlyFailedSubjobs = False
jobs(0).subjobs(0).force_status('failed')
jobs(0).resubmit()
sleep_until_completed(jobs(0))
assert jobs(0).subjobs(0).status == 'completed'
def test_d_SJResubmit_FailNotRequired(self):
"""
Resubmit when jobs are not required to have failed
"""
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'False')
from Ganga.GPI import jobs
# Test that resubmitting a job with SubJobXMLList subjobs doesn't stall
# Test them with all subjobs completed and resubmitOnlyFailedSubjobs = False
jobs(0).resubmit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(jobs(0))
# Test that resubmitting a subjob from SubJobXMLList that subjob doesn't stall
jobs(0).subjobs(0).resubmit()
jobs(0).subjobs(0).force_status('failed')
jobs(0).subjobs(0).resubmit()
sleep_until_completed(jobs(0))
assert jobs(0).subjobs(0).status == 'completed'
# Test that we can resubmit 1 subjob from the subjob itself when the subjob is failed
# resubmitOnlyFailedSubjobs = False
jobs(0).subjobs(0).force_status('failed')
jobs(0).resubmit()
sleep_until_completed(jobs(0))
assert jobs(0).subjobs(0).status == 'completed'
def test_ConfigDefaults(self):
from Ganga.GPI import Executable
from Ganga.Utility.Config import setConfigOption
setConfigOption('defaults_Executable', 'exe', 'echo2')
setConfigOption('defaults_Executable', 'args', ['Hello World2'])
a = Executable()
assert a.exe == 'echo2'
assert a.args == ['Hello World2']
setConfigOption('defaults_Executable', 'exe', '/bin/echo')
k = Executable()
assert k.exe == '/bin/echo'
assert k.args == ['Hello World2']
def test_ConfigDefaults(self):
from Ganga.GPI import Executable
from Ganga.Utility.Config import setConfigOption
setConfigOption('defaults_Executable', 'exe', 'echo2')
setConfigOption('defaults_Executable', 'args', ['Hello World2'])
a = Executable()
assert a.exe == 'echo2'
assert a.args == ['Hello World2']
setConfigOption('defaults_Executable', 'exe', '/bin/echo')
k = Executable()
assert k.exe == '/bin/echo'
assert k.args == ['Hello World2']
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
super(TestSavannah9779, self).setUp()
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def setUp(self):
super(TestSavannah32342, self).setUp()
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def start_ganga(gangadir_for_test, extra_opts=[]):
"""
Startup Ganga by calling the same set of 'safe' functions each time
Args:
gangadir_for_test (str): This is the directory which the GangaUnitTest is to be run, a new gangadir has been created per test to avoid collisions
extra_opts (list): A list of tuples which are used to pass command line style options to Ganga
"""
import Ganga.PACKAGE
Ganga.PACKAGE.standardSetup()
# End taken from the ganga binary
import Ganga.Runtime
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Start ganga by passing some options for unittesting
logger.info("Starting ganga")
logger.info("Parsing Command Line options")
this_argv = [
'ganga', # `argv[0]` is usually the name of the program so fake that here
]
# These are the default options for all test instances
# They can be overridden by extra_opts
default_opts = [
('Configuration', 'RUNTIME_PATH', 'GangaTest'),
('Configuration', 'gangadir', gangadir_for_test),
('Configuration', 'user', 'testframework'),
('Configuration', 'repositorytype', 'LocalXML'),
('Configuration', 'UsageMonitoringMSG', False), # Turn off spyware
('TestingFramework', 'ReleaseTesting', True),
('Queues', 'NumWorkerThreads', 2),
]
# FIXME Should we need to add the ability to load from a custom .ini file
# to configure tests without editting this?
# Actually parse the options
Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
Ganga.Runtime._prog.parseOptions()
# For all the default and extra options, we set the session value
from Ganga.Utility.Config import setConfigOption
for opt in default_opts + extra_opts:
setConfigOption(*opt)
# The configuration is currently created at module import and hence can't be
# regenerated.
# The values read in from any .ini file or from command line will change this
# but the configuration can't be obliterated and re-created. (yet, 16.06.16)
# Perform the configuration and bootstrap steps in ganga
logger.info("Parsing Configuration Options")
Ganga.Runtime._prog.configure()
logger.info("Initializing")
Ganga.Runtime._prog.initEnvironment()
logger.info("Bootstrapping")
Ganga.Runtime._prog.bootstrap(interactive=False)
# We need to test if the internal services need to be reinitialized
from Ganga.Core.InternalServices import Coordinator
if not Coordinator.servicesEnabled:
# Start internal services
logger.info("InternalServices restarting")
from Ganga.Core.InternalServices.Coordinator import enableInternalServices
enableInternalServices()
else:
logger.info("InternalServices still running")
# Adapted from the Coordinator class, check for the required credentials and stop if not found
# Hopefully stops us falling over due to no AFS access of something similar
from Ganga.Core.InternalServices import Coordinator
missing_cred = Coordinator.getMissingCredentials()
logger.info("Checking Credentials")
if missing_cred:
raise Exception("Failed due to missing credentials %s" % str(missing_cred))
# Make sure that all the config options are really set.
# Some from plugins may not have taken during startup
for opt in default_opts + extra_opts:
setConfigOption(*opt)
logger.info("Passing to Unittest")
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
super(TestSavannah10016, self).setUp()
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def _ganga_run_exitfuncs():
"""Run all exit functions from plugins and internal services in the correct order
Go over all plugins and internal services and call the appropriate shutdown functions in the correct order. Because
we want each shutdown function to be run (e.g. to make sure flushing is done) we put each call into a try..except
and report to the user before continuing.
"""
# Set the disk timeout to 1 sec, sacrifice stability for quicker exit
setConfigOption('Configuration', 'DiskIOTimeout', 1)
# Stop the monitoring loop from iterating further
if monitoring_component is not None:
try:
getStackTrace()
if monitoring_component.alive:
monitoring_component.disableMonitoring()
monitoring_component.stop()
monitoring_component.join()
except Exception as err:
logger.exception(
"Exception raised while stopping the monitoring: %s" % err)
# Stop the tasks system from running
try:
stopTasks()
except Exception as err:
logger.exception("Exception raised while stopping Tasks: %s" % err)
# purge the monitoring queues
try:
_purge_actions_queue()
stop_and_free_thread_pool()
except Exception as err:
logger.exception(
"Exception raised while purging monitoring queues: %s" % err)
# Freeze queues
try:
if _global_queues:
_global_queues.freeze()
except Exception as err:
logger.exception(
"Exception raised during freeze of Global Queues: %s" % err)
# shutdown the threads in the GangaThreadPool
try:
GangaThreadPool.getInstance().shutdown()
except Exception as err:
logger.exception(
"Exception raised during shutdown of GangaThreadPool: %s" % err)
# Shutdown queues
try:
logger.info(
"Stopping Job processing before shutting down Repositories")
shutDownQueues()
except Exception as err:
logger.exception(
"Exception raised while purging shutting down queues: %s" % err)
# shutdown the repositories
try:
logger.info("Shutting Down Ganga Repositories")
Repository_runtime.shutdown()
except Exception as err:
logger.exception(
"Exception raised while shutting down repositories: %s" % err)
# label services as disabled
Coordinator.servicesEnabled = False
# shutdown SessionLock
try:
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
except Exception as err:
logger.exception(
"Exception raised while shutting down SessionLocks: %s" % err)
# Shutdown stacktracer
if stacktracer._tracer:
try:
stacktracer.trace_stop()
except Exception as err:
logger.exception(
"Exception raised while stopping stack tracer: %s" % err)
# do final shutdown
if requires_shutdown is True:
try:
final_shutdown()
except Exception as err:
logger.exception(
"Exception raised while doing final shutdown: %s" % err)
# show any open files after everything's shutdown
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
def _ganga_run_exitfuncs():
"""run any registered exit functions
atexit._exithandlers is traversed based on the priority.
If no priority was registered for a given function
than the lowest priority is assumed (LIFO policy)
We keep the same functionality as in *atexit* bare module but
we run each exit handler inside a try..catch block to be sure all
the registered handlers are executed
"""
from Ganga.GPIDev.Base.Proxy import getName
#print("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.flush_all()
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'DiskIOTimeout', 1)
## Stop the Mon loop from iterating further!
from Ganga.Core import monitoring_component
if monitoring_component is not None:
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
getStackTrace()
monitoring_component.disableMonitoring()
## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
from Ganga.GPIDev.Lib.Tasks import stopTasks
stopTasks()
# Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
#from Ganga.Utility.Config import setConfigOption
#setConfigOption('Configuration', 'DiskIOTimeout', 3)
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
def priority_cmp(f1, f2):
"""
Sort the exit functions based on priority in reversed order
"""
# extract the priority number from the function element
p1 = f1[0][0]
p2 = f2[0][0]
# sort in reversed order
return cmp(p2, p1)
def add_priority(x):
"""
add a default priority to the functions not defining one (default priority=sys.maxint)
return a list containg ((priority,func),*targs,*kargs) elements
"""
import sys
func = x[0]
if isinstance(func, tuple) and len(x[0]) == 2:
return x
else:
new = [(sys.maxsize, func)]
new.extend(x[1:])
return new
atexit._exithandlers = map(add_priority, atexit._exithandlers)
atexit._exithandlers.sort(priority_cmp)
logger.info("Stopping Job processing before shutting down Repositories")
import inspect
while atexit._exithandlers:
(priority, func), targs, kargs = atexit._exithandlers.pop()
try:
if hasattr(func, 'im_class'):
for cls in inspect.getmro(func.__self__.__class__):
if getName(func) in cls.__dict__:
logger.debug(getName(cls) + " : " + getName(func))
func(*targs, **kargs)
else:
logger.debug("noclass : " + getName(func))
#print("%s" % str(func))
#print("%s" % str(inspect.getsourcefile(func)))
#func(*targs, **kargs)
## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
if str(inspect.getsourcefile(func)).find('Ganga') != -1:
func(*targs, **kargs)
except Exception as err:
s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (getName(func), str(err))
logger.debug(s)
try:
import os
logger.debug("%s" % os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(func)))) )
logger.debug("\n%s" % inspect.getsource(func))
except Exception as err2:
logger.debug("Error getting source code and failure reason: %s" % str(err2))
logger.info("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
from Ganga.Core.InternalServices import Coordinator
Coordinator.servicesEnabled = False
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
from Ganga.Utility.logging import requires_shutdown, final_shutdown
if requires_shutdown is True:
final_shutdown()
from Ganga.Runtime import bootstrap
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
def setUp(self):
super(TestRegistry, self).setUp()
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def _ganga_run_exitfuncs():
"""run any registered exit functions
atexit._exithandlers is traversed based on the priority.
If no priority was registered for a given function
than the lowest priority is assumed (LIFO policy)
We keep the same functionality as in *atexit* bare module but
we run each exit handler inside a try..catch block to be sure all
the registered handlers are executed
"""
from Ganga.GPIDev.Base.Proxy import getName
#print("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.flush_all()
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Set the disk timeout to 1 sec, sacrifice stability for quick-er exit
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'DiskIOTimeout', 1)
try:
from Ganga.GPI import queues
queues.lock()
except Exception as err:
logger.debug(
"This should only happen if Ganga filed to initialize correctly")
logger.debug("Err: %s" % str(err))
## Stop the Mon loop from iterating further!
from Ganga.Core import monitoring_component
if monitoring_component is not None:
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import getStackTrace
getStackTrace()
monitoring_component.disableMonitoring()
#monitoring_component.stop()
## This will stop the Registries flat but we may still have threads processing data!
#from Ganga.Core.InternalServices import Coordinator
#if Coordinator.servicesEnabled:
# Coordinator.disableInternalServices( shutdown = True )
## Stop the tasks system from running it's GangaThread before we get to the GangaThread shutdown section!
from Ganga.GPIDev.Lib.Tasks import stopTasks
stopTasks()
# Set the disk timeout to 3 sec, sacrifice stability for quick-er exit
#from Ganga.Utility.Config import setConfigOption
#setConfigOption('Configuration', 'DiskIOTimeout', 3)
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _purge_actions_queue, stop_and_free_thread_pool
_purge_actions_queue()
stop_and_free_thread_pool()
try:
from Ganga.GPI import queues
queues._purge_all()
except Exception as err:
logger.debug(
"This should only happen if Ganga filed to initialize correctly")
logger.debug("Err2: %s" % str(err))
def priority_cmp(f1, f2):
"""
Sort the exit functions based on priority in reversed order
"""
# extract the priority number from the function element
p1 = f1[0][0]
p2 = f2[0][0]
# sort in reversed order
return cmp(p2, p1)
def add_priority(x):
"""
add a default priority to the functions not defining one (default priority=sys.maxint)
return a list containg ((priority,func),*targs,*kargs) elements
"""
import sys
func = x[0]
if isinstance(func, tuple) and len(x[0]) == 2:
return x
else:
new = [(sys.maxsize, func)]
new.extend(x[1:])
return new
atexit._exithandlers = map(add_priority, atexit._exithandlers)
atexit._exithandlers.sort(priority_cmp)
logger.info("Stopping running tasks before shutting down Repositories")
import inspect
while atexit._exithandlers:
(priority, func), targs, kargs = atexit._exithandlers.pop()
try:
if hasattr(func, 'im_class'):
for cls in inspect.getmro(func.__self__.__class__):
if getName(func) in cls.__dict__:
logger.debug(getName(cls) + " : " + getName(func))
func(*targs, **kargs)
else:
logger.debug("noclass : " + getName(func))
#print("%s" % str(func))
#print("%s" % str(inspect.getsourcefile(func)))
#func(*targs, **kargs)
## This attempts to check for and remove externally defined shutdown functions which may interfere with the next steps!
if str(inspect.getsourcefile(func)).find('Ganga') != -1:
func(*targs, **kargs)
except Exception as err:
s = 'Cannot run one of the exit handlers: %s ... Cause: %s' % (
getName(func), str(err))
logger.debug(s)
try:
import os
logger.debug("%s" % os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(func)))))
logger.debug("\n%s" % inspect.getsource(func))
except Exception as err2:
logger.debug(
"Error getting source code and failure reason: %s" %
str(err2))
logger.info("Shutting Down Ganga Repositories")
from Ganga.Runtime import Repository_runtime
Repository_runtime.shutdown()
from Ganga.Core.InternalServices import Coordinator
Coordinator.servicesEnabled = False
from Ganga.Core.GangaRepository.SessionLock import removeGlobalSessionFiles, removeGlobalSessionFileHandlers
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
from Ganga.Utility.logging import requires_shutdown, final_shutdown
if requires_shutdown is True:
final_shutdown()
from Ganga.Runtime import bootstrap
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
def _ganga_run_exitfuncs():
"""Run all exit functions from plugins and internal services in the correct order
Go over all plugins and internal services and call the appropriate shutdown functions in the correct order. Because
we want each shutdown function to be run (e.g. to make sure flushing is done) we put each call into a try..except
and report to the user before continuing.
"""
# Set the disk timeout to 1 sec, sacrifice stability for quicker exit
setConfigOption('Configuration', 'DiskIOTimeout', 1)
# Stop the monitoring loop from iterating further
if monitoring_component is not None:
try:
getStackTrace()
if monitoring_component.alive:
monitoring_component.disableMonitoring()
monitoring_component.stop()
monitoring_component.join()
except Exception as err:
logger.exception("Exception raised while stopping the monitoring: %s" % err)
# Stop the tasks system from running
try:
stopTasks()
except Exception as err:
logger.exception("Exception raised while stopping Tasks: %s" % err)
# purge the monitoring queues
try:
_purge_actions_queue()
stop_and_free_thread_pool()
except Exception as err:
logger.exception("Exception raised while purging monitoring queues: %s" % err)
# Freeze queues
try:
if _global_queues:
_global_queues.freeze()
except Exception as err:
logger.exception("Exception raised during freeze of Global Queues: %s" % err)
# shutdown the threads in the GangaThreadPool
try:
GangaThreadPool.getInstance().shutdown()
except Exception as err:
logger.exception("Exception raised during shutdown of GangaThreadPool: %s" % err)
# Shutdown queues
try:
logger.info("Stopping Job processing before shutting down Repositories")
shutDownQueues()
except Exception as err:
logger.exception("Exception raised while purging shutting down queues: %s" % err)
# shutdown the repositories
try:
logger.info("Shutting Down Ganga Repositories")
Repository_runtime.shutdown()
except Exception as err:
logger.exception("Exception raised while shutting down repositories: %s" % err)
# label services as disabled
Coordinator.servicesEnabled = False
# clear the credential store
try:
CredentialStore.shutdown()
except Exception as err:
logger.exception("Exception raised while clearing the credential store: %s" % err)
# shutdown SessionLock
try:
removeGlobalSessionFileHandlers()
removeGlobalSessionFiles()
except Exception as err:
logger.exception("Exception raised while shutting down SessionLocks: %s" % err)
# Shutdown stacktracer
if stacktracer._tracer:
try:
stacktracer.trace_stop()
except Exception as err:
logger.exception("Exception raised while stopping stack tracer: %s" % err)
# do final shutdown
if requires_shutdown is True:
try:
final_shutdown()
except Exception as err:
logger.exception("Exception raised while doing final shutdown: %s" % err)
# show any open files after everything's shutdown
if bootstrap.DEBUGFILES or bootstrap.MONITOR_FILES:
bootstrap.printOpenFiles()
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
super(TestSavannah10016, self).setUp()
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def start_ganga(gangadir_for_test='$HOME/gangadir_testing', extra_opts=[]):
import sys
import os.path
file_path = os.path.dirname(os.path.realpath(__file__))
ganga_python_dir = os.path.join(file_path, '..', '..', '..')
ganga_python_dir = os.path.realpath(ganga_python_dir)
if len(sys.path) >= 1 and ganga_python_dir != sys.path[0]:
sys.path.insert(0, ganga_python_dir)
print("Adding: %s to Python Path\n" % ganga_python_dir)
import Ganga.PACKAGE
Ganga.PACKAGE.standardSetup()
# End taken from the ganga binary
import Ganga.Runtime
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Start ganga by passing some options for unittesting
logger.info("Starting ganga")
logger.info("Parsing Command Line options")
import Ganga.Runtime
this_argv = [
'ganga', # `argv[0]` is usually the name of the program so fake that here
]
# These are the default options for all test instances
# They can be overridden by extra_opts
default_opts = [
('Configuration', 'RUNTIME_PATH', 'GangaTest'),
('Configuration', 'gangadir', gangadir_for_test),
('Configuration', 'user', 'testframework'),
('Configuration', 'repositorytype', 'LocalXML'),
('TestingFramework', 'ReleaseTesting', True),
('Queues', 'NumWorkerThreads', 2),
]
# FIXME Should we need to add the ability to load from a custom .ini file
# to configure tests without editting this?
# Actually parse the options
Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
Ganga.Runtime._prog.parseOptions()
# Determine if ganga has actually finished initializing...
# This is here to protect against the startGanga being called on an
# initialized ganga environment
try:
do_config = not Ganga.Utility.Config.Config._after_bootstrap
except:
do_config = True
# For all the default and extra options, we set the session value
from Ganga.Utility.Config import setConfigOption
for opt in default_opts + extra_opts:
setConfigOption(*opt)
if do_config:
# Perform the configuration and bootstrap steps in ganga
logger.info("Parsing Configuration Options")
Ganga.Runtime._prog.configure()
logger.info("Initializing")
Ganga.Runtime._prog.initEnvironment(opt_rexec=False)
else:
# We need to test if the internal services need to be reinitialized
from Ganga.Core.InternalServices import Coordinator
if not Coordinator.servicesEnabled:
# Start internal services
logger.info("InternalServices restarting")
from Ganga.GPI import reactivate
reactivate()
else:
logger.info("InternalServices still running")
# The queues are shut down by the atexit handlers so we need to start them here
from Ganga.Core.GangaThread.WorkerThreads import startUpQueues
startUpQueues()
logger.info("Bootstrapping")
Ganga.Runtime._prog.bootstrap(interactive=False)
# Adapted from the Coordinator class, check for the required credentials and stop if not found
# Hopefully stops us falling over due to no AFS access of something similar
from Ganga.Core.InternalServices import Coordinator
missing_cred = Coordinator.getMissingCredentials()
logger.info("Checking Credentials")
if missing_cred:
raise Exception("Failed due to missing credentials %s" % str(missing_cred))
logger.info("Passing to Unittest")
def setUp(self):
extra_opts = [('Queues', 'NumWorkerThreads', global_num_threads)]
super(TestQueuedSubmit, self).setUp(extra_opts=extra_opts)
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
super(TestSelect, self).setUp()
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
extra_opts = [('Registry', 'AutoFlusherWaitTime', 10)]
super(TestSJXMLGenAndLoad, self).setUp(extra_opts=extra_opts)
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
def setUp(self):
"""Make sure that the Job object isn't destroyed between tests"""
super(TestSJXMLCorruption, self).setUp()
from Ganga.Utility.Config import setConfigOption
setConfigOption('TestingFramework', 'AutoCleanup', 'False')
setConfigOption('Configuration', 'AutoStartReg', global_AutoStartReg)
def start_ganga(gangadir_for_test, extra_opts=[]):
"""
Startup Ganga by calling the same set of 'safe' functions each time
Args:
gangadir_for_test (str): This is the directory which the GangaUnitTest is to be run, a new gangadir has been created per test to avoid collisions
extra_opts (list): A list of tuples which are used to pass command line style options to Ganga
"""
import Ganga.PACKAGE
Ganga.PACKAGE.standardSetup()
# End taken from the ganga binary
import Ganga.Runtime
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Start ganga by passing some options for unittesting
logger.info("Starting ganga")
logger.info("Parsing Command Line options")
this_argv = [
'ganga', # `argv[0]` is usually the name of the program so fake that here
]
# These are the default options for all test instances
# They can be overridden by extra_opts
default_opts = [
('Configuration', 'RUNTIME_PATH', 'GangaTest'),
('Configuration', 'gangadir', gangadir_for_test),
('Configuration', 'user', 'testframework'),
('Configuration', 'repositorytype', 'LocalXML'),
('Configuration', 'UsageMonitoringMSG', False), # Turn off spyware
('TestingFramework', 'ReleaseTesting', True),
('Queues', 'NumWorkerThreads', 2),
]
# FIXME Should we need to add the ability to load from a custom .ini file
# to configure tests without editting this?
# Actually parse the options
Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
Ganga.Runtime._prog.default_config_file = ganga_config_file
Ganga.Runtime._prog.parseOptions()
# For all the default and extra options, we set the session value
from Ganga.Utility.Config import setConfigOption
for opt in default_opts + extra_opts:
setConfigOption(*opt)
# The configuration is currently created at module import and hence can't be
# regenerated.
# The values read in from any .ini file or from command line will change this
# but the configuration can't be obliterated and re-created. (yet, 16.06.16)
# Perform the configuration and bootstrap steps in ganga
logger.info("Parsing Configuration Options")
Ganga.Runtime._prog.configure()
logger.info("Initializing")
Ganga.Runtime._prog.initEnvironment()
logger.info("Bootstrapping")
Ganga.Runtime._prog.bootstrap(interactive=False)
# We need to test if the internal services need to be reinitialized
from Ganga.Core.InternalServices import Coordinator
if not Coordinator.servicesEnabled:
# Start internal services
logger.info("InternalServices restarting")
from Ganga.Core.InternalServices.Coordinator import enableInternalServices
enableInternalServices()
else:
logger.info("InternalServices still running")
# Adapted from the Coordinator class, check for the required credentials and stop if not found
# Hopefully stops us falling over due to no AFS access of something similar
from Ganga.Core.InternalServices import Coordinator
missing_cred = Coordinator.getMissingCredentials()
logger.info("Checking Credentials")
if missing_cred:
raise Exception("Failed due to missing credentials %s" %
str(missing_cred))
# Make sure that all the config options are really set.
# Some from plugins may not have taken during startup
for opt in default_opts + extra_opts:
setConfigOption(*opt)
logger.info("Passing to Unittest")
Ganga.Runtime._prog.parseOptions()
# Determine if ganga has actually finished initializing...
# This is here to protect against the startGanga being called on an
# initialized ganga environment
try:
doConfig = not Ganga.Utility.Config.Config._after_bootstrap
except:
doConfig = True
if doConfig:
# Perform the configuration and bootstrap steps in ganga
logger.info("Parsing Configuration Options")
Ganga.Runtime._prog.configure()
from Ganga.Utility.Config import setConfigOption
setConfigOption('PollThread', 'forced_shutdown_policy', 'batch')
logger.info("Initializing")
Ganga.Runtime._prog.initEnvironment()
logger.info("Bootstrapping")
Ganga.Runtime._prog.bootstrap()
else:
# No need to perform the bootstrap but we need to test if the internal
# services need to be reinitialized
from Ganga.Utility.Config import setConfigOption
setConfigOption('PollThread', 'forced_shutdown_policy', 'batch')
from Ganga.Core.InternalServices import Coordinator
if not Coordinator.servicesEnabled:
# Start internal services
logger.info("InternalServices restarting")
def testing_cb(t_total, critical_thread_ids, non_critical_thread_ids):
