def runMonitoring(self, jobs=None, steps=1, timeout=60, _loadCredentials=False):
"""
Enable/Run the monitoring loop and wait for the monitoring steps completion.
Parameters:
steps: number of monitoring steps to run
timeout: how long to wait for monitor steps termination (seconds)
jobs: a registry slice to be monitored (None -> all jobs), it may be passed by the user so ._impl is stripped if needed
Return:
False, if the loop cannot be started or the timeout occured while waiting for monitoring termination
True, if the monitoring steps were successfully executed
Note:
This method is meant to be used in Ganga scripts to request monitoring on demand.
"""
log.debug("runMonitoring")
if not isType(steps, int) and steps < 0:
log.warning("The number of monitor steps should be a positive (non-zero) integer")
return False
if not self.alive:
log.error("Cannot run the monitoring loop. It has already been stopped")
return False
# we don not allow the user's request the monitoring loop while the
# internal services are stopped
if not Coordinator.servicesEnabled:
log.error("Cannot run the monitoring loop."
"The internal services are disabled (check your credentials or available disk space)")
return False
# if the monitoring is disabled (e.g. scripts)
if not self.enabled:
# and there are some required cred which are missing
# (the monitoring loop does not monitor the credentials so we need to check 'by hand' here)
if _loadCredentials is True:
_missingCreds = Coordinator.getMissingCredentials()
else:
_missingCreds = False
if _missingCreds:
log.error("Cannot run the monitoring loop. The following credentials are required: %s" % _missingCreds)
return False
#log.debug("jobs: %s" % str(jobs))
#log.debug("self.__mainLoopCond: %s" % str(self.__mainLoopCond))
with self.__mainLoopCond:
log.debug('Monitoring loop lock acquired. Enabling mon loop')
if self.enabled or self.__isInProgress():
log.error("The monitoring loop is already running.")
return False
if jobs is not None:
m_jobs = jobs
# additional check if m_jobs is really a registry slice
# the underlying code is not prepared to handle correctly the
# situation if it is not
from Ganga.GPIDev.Lib.Registry.RegistrySlice import RegistrySlice
if not isType(m_jobs, RegistrySlice):
log.warning('runMonitoring: jobs argument must be a registry slice such as a result of jobs.select() or jobs[i1:i2]')
return False
self.registry = m_jobs
#log.debug("m_jobs: %s" % str(m_jobs))
self.makeUpdateJobStatusFunction()
log.debug("Enable Loop, Clear Iterators and setCallbackHook")
# enable mon loop
self.enabled = True
# set how many steps to run
self.steps = steps
# enable job list iterators
self.stopIter.clear()
# Start backend update timeout checking.
self.setCallbackHook(self.updateDict_ts.timeoutCheck, {}, True)
log.debug("Waking up Main Loop")
# wake up the mon loop
self.__mainLoopCond.notifyAll()
log.debug("Waiting to execute steps")
# wait to execute the steps
self.__monStepsTerminatedEvent.wait()
self.__monStepsTerminatedEvent.clear()
log.debug("Test for timeout")
# wait the steps to be executed or timeout to occur
if not self.__awaitTermination(timeout):
log.warning("Monitoring loop started but did not complete in the given timeout.")
# force loops termination
self.stopIter.set()
return False
return True
def start_ganga(gangadir_for_test, extra_opts=[]):
"""
Startup Ganga by calling the same set of 'safe' functions each time
Args:
gangadir_for_test (str): This is the directory which the GangaUnitTest is to be run, a new gangadir has been created per test to avoid collisions
extra_opts (list): A list of tuples which are used to pass command line style options to Ganga
"""
import Ganga.PACKAGE
Ganga.PACKAGE.standardSetup()
# End taken from the ganga binary
import Ganga.Runtime
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Start ganga by passing some options for unittesting
logger.info("Starting ganga")
logger.info("Parsing Command Line options")
this_argv = [
'ganga', # `argv[0]` is usually the name of the program so fake that here
]
# These are the default options for all test instances
# They can be overridden by extra_opts
default_opts = [
('Configuration', 'RUNTIME_PATH', 'GangaTest'),
('Configuration', 'gangadir', gangadir_for_test),
('Configuration', 'user', 'testframework'),
('Configuration', 'repositorytype', 'LocalXML'),
('Configuration', 'UsageMonitoringMSG', False), # Turn off spyware
('TestingFramework', 'ReleaseTesting', True),
('Queues', 'NumWorkerThreads', 2),
]
# FIXME Should we need to add the ability to load from a custom .ini file
# to configure tests without editting this?
# Actually parse the options
Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
Ganga.Runtime._prog.default_config_file = ganga_config_file
Ganga.Runtime._prog.parseOptions()
# For all the default and extra options, we set the session value
from Ganga.Utility.Config import setConfigOption
for opt in default_opts + extra_opts:
setConfigOption(*opt)
# The configuration is currently created at module import and hence can't be
# regenerated.
# The values read in from any .ini file or from command line will change this
# but the configuration can't be obliterated and re-created. (yet, 16.06.16)
# Perform the configuration and bootstrap steps in ganga
logger.info("Parsing Configuration Options")
Ganga.Runtime._prog.configure()
logger.info("Initializing")
Ganga.Runtime._prog.initEnvironment()
logger.info("Bootstrapping")
Ganga.Runtime._prog.bootstrap(interactive=False)
# We need to test if the internal services need to be reinitialized
from Ganga.Core.InternalServices import Coordinator
if not Coordinator.servicesEnabled:
# Start internal services
logger.info("InternalServices restarting")
from Ganga.Core.InternalServices.Coordinator import enableInternalServices
enableInternalServices()
else:
logger.info("InternalServices still running")
# Adapted from the Coordinator class, check for the required credentials and stop if not found
# Hopefully stops us falling over due to no AFS access of something similar
from Ganga.Core.InternalServices import Coordinator
missing_cred = Coordinator.getMissingCredentials()
logger.info("Checking Credentials")
if missing_cred:
raise Exception("Failed due to missing credentials %s" %
str(missing_cred))
# Make sure that all the config options are really set.
# Some from plugins may not have taken during startup
for opt in default_opts + extra_opts:
setConfigOption(*opt)
logger.info("Passing to Unittest")
def start_ganga(gangadir_for_test, extra_opts=[]):
"""
Startup Ganga by calling the same set of 'safe' functions each time
Args:
gangadir_for_test (str): This is the directory which the GangaUnitTest is to be run, a new gangadir has been created per test to avoid collisions
extra_opts (list): A list of tuples which are used to pass command line style options to Ganga
"""
import Ganga.PACKAGE
Ganga.PACKAGE.standardSetup()
# End taken from the ganga binary
import Ganga.Runtime
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Start ganga by passing some options for unittesting
logger.info("Starting ganga")
logger.info("Parsing Command Line options")
this_argv = [
'ganga', # `argv[0]` is usually the name of the program so fake that here
]
# These are the default options for all test instances
# They can be overridden by extra_opts
default_opts = [
('Configuration', 'RUNTIME_PATH', 'GangaTest'),
('Configuration', 'gangadir', gangadir_for_test),
('Configuration', 'user', 'testframework'),
('Configuration', 'repositorytype', 'LocalXML'),
('Configuration', 'UsageMonitoringMSG', False), # Turn off spyware
('TestingFramework', 'ReleaseTesting', True),
('Queues', 'NumWorkerThreads', 2),
]
# FIXME Should we need to add the ability to load from a custom .ini file
# to configure tests without editting this?
# Actually parse the options
Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
Ganga.Runtime._prog.parseOptions()
# For all the default and extra options, we set the session value
from Ganga.Utility.Config import setConfigOption
for opt in default_opts + extra_opts:
setConfigOption(*opt)
# The configuration is currently created at module import and hence can't be
# regenerated.
# The values read in from any .ini file or from command line will change this
# but the configuration can't be obliterated and re-created. (yet, 16.06.16)
# Perform the configuration and bootstrap steps in ganga
logger.info("Parsing Configuration Options")
Ganga.Runtime._prog.configure()
logger.info("Initializing")
Ganga.Runtime._prog.initEnvironment()
logger.info("Bootstrapping")
Ganga.Runtime._prog.bootstrap(interactive=False)
# We need to test if the internal services need to be reinitialized
from Ganga.Core.InternalServices import Coordinator
if not Coordinator.servicesEnabled:
# Start internal services
logger.info("InternalServices restarting")
from Ganga.Core.InternalServices.Coordinator import enableInternalServices
enableInternalServices()
else:
logger.info("InternalServices still running")
# Adapted from the Coordinator class, check for the required credentials and stop if not found
# Hopefully stops us falling over due to no AFS access of something similar
from Ganga.Core.InternalServices import Coordinator
missing_cred = Coordinator.getMissingCredentials()
logger.info("Checking Credentials")
if missing_cred:
raise Exception("Failed due to missing credentials %s" % str(missing_cred))
# Make sure that all the config options are really set.
# Some from plugins may not have taken during startup
for opt in default_opts + extra_opts:
setConfigOption(*opt)
logger.info("Passing to Unittest")
def runMonitoring(self,
jobs=None,
steps=1,
timeout=60,
_loadCredentials=False):
"""
Enable/Run the monitoring loop and wait for the monitoring steps completion.
Parameters:
steps: number of monitoring steps to run
timeout: how long to wait for monitor steps termination (seconds)
jobs: a registry slice to be monitored (None -> all jobs), it may be passed by the user so ._impl is stripped if needed
Return:
False, if the loop cannot be started or the timeout occured while waiting for monitoring termination
True, if the monitoring steps were successfully executed
Note:
This method is meant to be used in Ganga scripts to request monitoring on demand.
"""
log.debug("runMonitoring")
if not isType(steps, int) and steps < 0:
log.warning(
"The number of monitor steps should be a positive (non-zero) integer"
)
return False
if not self.alive:
log.error(
"Cannot run the monitoring loop. It has already been stopped")
return False
# we don not allow the user's request the monitoring loop while the
# internal services are stopped
if not Coordinator.servicesEnabled:
log.error(
"Cannot run the monitoring loop."
"The internal services are disabled (check your credentials or available disk space)"
)
return False
# if the monitoring is disabled (e.g. scripts)
if not self.enabled:
# and there are some required cred which are missing
# (the monitoring loop does not monitor the credentials so we need to check 'by hand' here)
if _loadCredentials is True:
_missingCreds = Coordinator.getMissingCredentials()
else:
_missingCreds = False
if _missingCreds:
log.error(
"Cannot run the monitoring loop. The following credentials are required: %s"
% _missingCreds)
return False
#log.debug("jobs: %s" % str(jobs))
#log.debug("self.__mainLoopCond: %s" % str(self.__mainLoopCond))
with self.__mainLoopCond:
log.debug('Monitoring loop lock acquired. Enabling mon loop')
if self.enabled or self.__isInProgress():
log.error("The monitoring loop is already running.")
return False
if jobs is not None:
m_jobs = jobs
# additional check if m_jobs is really a registry slice
# the underlying code is not prepared to handle correctly the
# situation if it is not
from Ganga.GPIDev.Lib.Registry.RegistrySlice import RegistrySlice
if not isType(m_jobs, RegistrySlice):
log.warning(
'runMonitoring: jobs argument must be a registry slice such as a result of jobs.select() or jobs[i1:i2]'
)
return False
self.registry = m_jobs
#log.debug("m_jobs: %s" % str(m_jobs))
self.makeUpdateJobStatusFunction()
log.debug("Enable Loop, Clear Iterators and setCallbackHook")
# enable mon loop
self.enabled = True
# set how many steps to run
self.steps = steps
# enable job list iterators
self.stopIter.clear()
# Start backend update timeout checking.
self.setCallbackHook(updateDict_ts.timeoutCheck, {}, True)
log.debug("Waking up Main Loop")
# wake up the mon loop
self.__mainLoopCond.notifyAll()
log.debug("Waiting to execute steps")
# wait to execute the steps
self.__monStepsTerminatedEvent.wait()
self.__monStepsTerminatedEvent.clear()
log.debug("Test for timeout")
# wait the steps to be executed or timeout to occur
if not self.__awaitTermination(timeout):
log.warning(
"Monitoring loop started but did not complete in the given timeout."
)
# force loops termination
self.stopIter.set()
return False
return True
def start_ganga(gangadir_for_test='$HOME/gangadir_testing', extra_opts=[]):
import sys
import os.path
file_path = os.path.dirname(os.path.realpath(__file__))
ganga_python_dir = os.path.join(file_path, '..', '..', '..')
ganga_python_dir = os.path.realpath(ganga_python_dir)
if len(sys.path) >= 1 and ganga_python_dir != sys.path[0]:
sys.path.insert(0, ganga_python_dir)
print("Adding: %s to Python Path\n" % ganga_python_dir)
import Ganga.PACKAGE
Ganga.PACKAGE.standardSetup()
# End taken from the ganga binary
import Ganga.Runtime
from Ganga.Utility.logging import getLogger
logger = getLogger()
# Start ganga by passing some options for unittesting
logger.info("Starting ganga")
logger.info("Parsing Command Line options")
import Ganga.Runtime
this_argv = [
'ganga', # `argv[0]` is usually the name of the program so fake that here
]
# These are the default options for all test instances
# They can be overridden by extra_opts
default_opts = [
('Configuration', 'RUNTIME_PATH', 'GangaTest'),
('Configuration', 'gangadir', gangadir_for_test),
('Configuration', 'user', 'testframework'),
('Configuration', 'repositorytype', 'LocalXML'),
('TestingFramework', 'ReleaseTesting', True),
('Queues', 'NumWorkerThreads', 2),
]
# FIXME Should we need to add the ability to load from a custom .ini file
# to configure tests without editting this?
# Actually parse the options
Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
Ganga.Runtime._prog.parseOptions()
# Determine if ganga has actually finished initializing...
# This is here to protect against the startGanga being called on an
# initialized ganga environment
try:
do_config = not Ganga.Utility.Config.Config._after_bootstrap
except:
do_config = True
# For all the default and extra options, we set the session value
from Ganga.Utility.Config import setConfigOption
for opt in default_opts + extra_opts:
setConfigOption(*opt)
if do_config:
# Perform the configuration and bootstrap steps in ganga
logger.info("Parsing Configuration Options")
Ganga.Runtime._prog.configure()
logger.info("Initializing")
Ganga.Runtime._prog.initEnvironment(opt_rexec=False)
else:
# We need to test if the internal services need to be reinitialized
from Ganga.Core.InternalServices import Coordinator
if not Coordinator.servicesEnabled:
# Start internal services
logger.info("InternalServices restarting")
from Ganga.GPI import reactivate
reactivate()
else:
logger.info("InternalServices still running")
# The queues are shut down by the atexit handlers so we need to start them here
from Ganga.Core.GangaThread.WorkerThreads import startUpQueues
startUpQueues()
logger.info("Bootstrapping")
Ganga.Runtime._prog.bootstrap(interactive=False)
# Adapted from the Coordinator class, check for the required credentials and stop if not found
# Hopefully stops us falling over due to no AFS access of something similar
from Ganga.Core.InternalServices import Coordinator
missing_cred = Coordinator.getMissingCredentials()
logger.info("Checking Credentials")
if missing_cred:
raise Exception("Failed due to missing credentials %s" % str(missing_cred))
logger.info("Passing to Unittest")
def start_ganga():
import sys
import os.path
file_path = os.path.dirname(os.path.realpath(__file__))
ganga_python_dir = os.path.join(file_path, '..', '..', '..')
ganga_python_dir = os.path.realpath(ganga_python_dir)
sys.path.insert(0, ganga_python_dir)
print("Adding: %s to Python Path\n" % ganga_python_dir)
sys.path.insert(0, ganga_python_dir)
import Ganga.PACKAGE
Ganga.PACKAGE.standardSetup()
# End taken from the ganga binary
import Ganga.Runtime
import Ganga.Utility.logging
logger = Ganga.Utility.logging.getLogger()
# Start ganga by passing some options for unittesting
logger.info("Starting ganga")
logger.info("Parsing Command Line options")
import Ganga.Runtime
this_argv = [
'ganga', # `argv[0]` is usually the name of the program so fake that here
'-o[Configuration]RUNTIME_PATH=GangaTest',
'-o[Configuration]user=testframework',
'-o[Configuration]gangadir=$HOME/gangadir_testing',
'-o[Configuration]repositorytype=LocalXML',
'-o[PollThread]autostart_monThreads=False',
'-o[TestingFramework]ReleaseTesting=True',
]
# FIXME Should we need to add the ability to load from a custom .ini file
# to configure tests without editting this?
# Actually parse the options
Ganga.Runtime._prog = Ganga.Runtime.GangaProgram(argv=this_argv)
Ganga.Runtime._prog.parseOptions()
# Determine if ganga has actually finished initializing...
# This is here to protect against the startGanga being called on an
# initialized ganga environment
try:
do_config = not Ganga.Utility.Config.Config._after_bootstrap
except:
do_config = True
if do_config:
# Perform the configuration and bootstrap steps in ganga
logger.info("Parsing Configuration Options")
Ganga.Runtime._prog.configure()
logger.info("Initializing")
Ganga.Runtime._prog.initEnvironment(opt_rexec=False)
else:
# We need to test if the internal services need to be reinitialized
from Ganga.Core.InternalServices import Coordinator
if not Coordinator.servicesEnabled:
# Start internal services
logger.info("InternalServices restarting")
from Ganga.GPI import reactivate
reactivate()
else:
logger.info("InternalServices still running")
logger.info("Bootstrapping")
Ganga.Runtime._prog.bootstrap(interactive=False)
# [PollThread]autostart_monThreads=False has turned this off being done automatically.
# The thread pool is emptied by _ganga_run_exitfuncs
from Ganga.Core.MonitoringComponent.Local_GangaMC_Service import _makeThreadPool
_makeThreadPool()
# Adapted from the Coordinator class, check for the required credentials and stop if not found
# Hopefully stops us falling over due to no AFS access of something similar
from Ganga.Core.InternalServices import Coordinator
missing_cred = Coordinator.getMissingCredentials()
logger.info("Checking Credentials")
if missing_cred:
raise Exception("Failed due to missing credentials %s" %
str(missing_cred))
logger.info("Passing to Unittest")
logger.info("InternalServices restarting")
def testing_cb(t_total, critical_thread_ids, non_critical_thread_ids):
return True
from Ganga.Core.GangaThread import GangaThreadPool
thread_pool = GangaThreadPool.getInstance()
thread_pool.shutdown(should_wait_cb=testing_cb)
from Ganga.GPI import reactivate
reactivate()
else:
logger.info("InternalServices still running")
# Adapted from the Coordinator class, check for the required credentials and stop if not found
# Hopefully stops us falling over due to no AFS access of something similar
from Ganga.Core.InternalServices import Coordinator
missing_cred = Coordinator.getMissingCredentials()
logger.info("Checking Credentials")
if missing_cred:
raise Exception("Failed due to missing credentials %s" %
str(missing_cred))
logger.info("Passing to Unittest")
def stopGanga():
import Ganga.Utility.logging
logger = Ganga.Utility.logging.getLogger()
