def __init__(self, runid, configFileName, repository=None):
# _locked: a container for data to be shared across threads that
# have access to this object.
self._locked = SharedData.SharedData(False, {
"running": False,
"done": False
})
# the run id for this production
self.runid = runid
# once the workflows that make up this production is created we will
# cache them here
self._workflowManagers = None
# a list of workflow Monitors
self._workflowMonitors = []
# the cached ProductionRunConfigurator instance
self._productionRunConfigurator = None
# the full path the configuration
self.fullConfigFilePath = ""
if os.path.isabs(configFileName):
self.fullConfigFilePath = configFileName
else:
self.fullConfigFilePath = os.path.join(os.path.realpath('.'),
configFileName)
# create Production configuration
self.config = ProductionConfig()
# load the production config object
self.config.load(self.fullConfigFilePath)
# the repository location
self.repository = repository
# determine repository location
if not self.repository:
self.repository = self.config.production.repositoryDirectory
if not self.repository:
self.repository = "."
else:
self.repository = EnvString.resolve(self.repository)
# shutdown thread
self._sdthread = None
def __init__(self, runid, configFileName, repository=None):
# _locked: a container for data to be shared across threads that
# have access to this object.
self._locked = SharedData(False,
{"running": False, "done": False})
## the run id for this production
self.runid = runid
# once the workflows that make up this production is created we will
# cache them here
self._workflowManagers = None
# a list of workflow Monitors
self._workflowMonitors = []
# a list of logger managers
self._loggerManagers = []
# the cached ProductionRunConfigurator instance
self._productionRunConfigurator = None
## the full path the configuration
self.fullConfigFilePath = ""
if os.path.isabs(configFileName) == True:
self.fullConfigFilePath = configFileName
else:
self.fullConfigFilePath = os.path.join(os.path.realpath('.'), configFileName)
## create Production configuration
self.config = ProductionConfig()
# load the production config object
self.config.load(self.fullConfigFilePath)
## the repository location
self.repository = repository
# determine repository location
if not self.repository:
self.repository = self.config.production.repositoryDirectory
if not self.repository:
self.repository = "."
else:
self.repository = EnvString.resolve(self.repository)
# XXX - Check to see if we need to do this still.
# do a little sanity checking on the repository before we continue.
#if not os.path.exists(self.repository):
# raise RuntimeError("specified repository " + self.repository + ": directory not found");
#if not os.path.isdir(self.repository):
# raise RuntimeError("specified repository "+ self.repository + ": not a directory");
# shutdown thread
self._sdthread = None
def __init__(self,
runid,
configFile,
repository=None,
workflowVerbosity=None):
log.debug("ProductionRunConfigurator:__init__")
# run id for this production
self.runid = runid
self._prodConfigFile = configFile
# production configuration
self.prodConfig = ProductionConfig()
self.prodConfig.load(configFile)
# location of the repository
self.repository = repository
# verbosity level for this workflow
self.workflowVerbosity = workflowVerbosity
self._provSetup = None
# provenance dictionary
self.provenanceDict = {}
self._wfnames = None
# cache the database configurators for checking the configuraiton.
self._databaseConfigurators = []
# these are config settings which can be overriden from what they
# are in the workflow policies.
# dictionary of configuration override values
self.configOverrides = dict()
production = self.prodConfig.production
if production.logThreshold is not None:
self.configOverrides[
"execute.logThreshold"] = production.logThreshold
def __init__(self, runid, configFile, repository=None, workflowVerbosity=None):
log.debug("ProductionRunConfigurator:__init__")
## run id for this production
self.runid = runid
self._prodConfigFile = configFile
## production configuration
self.prodConfig = ProductionConfig()
self.prodConfig.load(configFile)
## location of the repostiry
self.repository = repository
## verbosity level for this workflow
self.workflowVerbosity = workflowVerbosity
self._provSetup = None
## provenance dictionary
self.provenanceDict = {}
self._wfnames = None
# cache the database configurators for checking the configuraiton.
self._databaseConfigurators = []
# logger managers
self._loggerManagers = []
## hostname of the event broker
self.eventBrokerHost = None
# these are config settings which can be overriden from what they
# are in the workflow policies.
## dictionary of configuration override values
self.configOverrides = dict()
production = self.prodConfig.production
if production.eventBrokerHost != None:
self.eventBrokerHost = production.eventBrokerHost
self.configOverrides["execute.eventBrokerHost"] = production.eventBrokerHost
if production.logThreshold != None:
self.configOverrides["execute.logThreshold"] = production.logThreshold
if production.productionShutdownTopic != None:
self.configOverrides["execute.shutdownTopic"] = production.productionShutdownTopic
def __init__(self, runid, configFileName, repository=None):
# _locked: a container for data to be shared across threads that
# have access to this object.
self._locked = SharedData.SharedData(False, {"running": False, "done": False})
# the run id for this production
self.runid = runid
# once the workflows that make up this production is created we will
# cache them here
self._workflowManagers = None
# a list of workflow Monitors
self._workflowMonitors = []
# the cached ProductionRunConfigurator instance
self._productionRunConfigurator = None
# the full path the configuration
self.fullConfigFilePath = ""
if os.path.isabs(configFileName):
self.fullConfigFilePath = configFileName
else:
self.fullConfigFilePath = os.path.join(os.path.realpath('.'), configFileName)
# create Production configuration
self.config = ProductionConfig()
# load the production config object
self.config.load(self.fullConfigFilePath)
# the repository location
self.repository = repository
# determine repository location
if not self.repository:
self.repository = self.config.production.repositoryDirectory
if not self.repository:
self.repository = "."
else:
self.repository = EnvString.resolve(self.repository)
# shutdown thread
self._sdthread = None
def __init__(self, runid, configFile, repository=None, workflowVerbosity=None):
log.debug("ProductionRunConfigurator:__init__")
# run id for this production
self.runid = runid
self._prodConfigFile = configFile
# production configuration
self.prodConfig = ProductionConfig()
self.prodConfig.load(configFile)
# location of the repository
self.repository = repository
# verbosity level for this workflow
self.workflowVerbosity = workflowVerbosity
self._provSetup = None
# provenance dictionary
self.provenanceDict = {}
self._wfnames = None
# cache the database configurators for checking the configuraiton.
self._databaseConfigurators = []
# these are config settings which can be overriden from what they
# are in the workflow policies.
# dictionary of configuration override values
self.configOverrides = dict()
production = self.prodConfig.production
if production.logThreshold is not None:
self.configOverrides["execute.logThreshold"] = production.logThreshold
class ProductionRunManager(object):
"""In charge of launching, monitoring, managing, and stopping a production run
Parameters
----------
runid : `str`
name of the run
configFileName : `Config`
production run config file
repository : `str`, optional
the config repository to assume; this will override the value in the config file
"""
def __init__(self, runid, configFileName, repository=None):
# _locked: a container for data to be shared across threads that
# have access to this object.
self._locked = SharedData.SharedData(False, {"running": False, "done": False})
# the run id for this production
self.runid = runid
# once the workflows that make up this production is created we will
# cache them here
self._workflowManagers = None
# a list of workflow Monitors
self._workflowMonitors = []
# the cached ProductionRunConfigurator instance
self._productionRunConfigurator = None
# the full path the configuration
self.fullConfigFilePath = ""
if os.path.isabs(configFileName):
self.fullConfigFilePath = configFileName
else:
self.fullConfigFilePath = os.path.join(os.path.realpath('.'), configFileName)
# create Production configuration
self.config = ProductionConfig()
# load the production config object
self.config.load(self.fullConfigFilePath)
# the repository location
self.repository = repository
# determine repository location
if not self.repository:
self.repository = self.config.production.repositoryDirectory
if not self.repository:
self.repository = "."
else:
self.repository = EnvString.resolve(self.repository)
# shutdown thread
self._sdthread = None
def getRunId(self):
"""Accessor to return the run id for this production run
Returns
-------
The runid of this production run
"""
return self.runid
def configure(self, workflowVerbosity=None):
"""Configure this production run
Parameters
----------
workflowVerbosity : `int`
The verbosity to pass down to configured workflows and the pipelines they run.
Raises
------
`ConfigurationError`
If any error arises during configuration or while checking the configuration.
Notes
-----
If the production was already configured, this call will be ignored and will not be reconfigured.
"""
if self._productionRunConfigurator:
log.info("production has already been configured.")
return
# lock this branch of code
try:
self._locked.acquire()
# TODO - SRP
self._productionRunConfigurator = self.createConfigurator(self.runid,
self.fullConfigFilePath)
workflowManagers = self._productionRunConfigurator.configure(workflowVerbosity)
self._workflowManagers = {"__order": []}
for wfm in workflowManagers:
self._workflowManagers["__order"].append(wfm)
self._workflowManagers[wfm.getName()] = wfm
finally:
self._locked.release()
def runProduction(self, skipConfigCheck=False, workflowVerbosity=None):
"""Run the entire production
Parameters
----------
skipConfigCheck : `bool`
Skips configuration checks, if True
workflowVerbosity: `int`, optional
overrides the config-specified logger verbosity
Raises
------
`ConfigurationError`
if any error arises during configuration or while checking the configuration.
Notes
-----
The skipConfigCheck parameter will be overridden by configCheckCare config parameter, if it exists.
The workflowVerbosity parameter will only be used if the run has not already been configured via
configure().
"""
log.debug("Running production: %s", self.runid)
if not self.isRunnable():
if self.isRunning():
log.info("Production Run %s is already running" % self.runid)
if self.isDone():
log.info("Production Run %s has already run; start with new runid" % self.runid)
return False
# set configuration check care level.
# Note: this is not a sanctioned pattern; should be replaced with use
# of default config.
checkCare = 1
if self.config.production.configCheckCare != 0:
checkCare = self.config.production.configCheckCare
if checkCare < 0:
skipConfigCheck = True
# lock this branch of code
try:
self._locked.acquire()
self._locked.running = True
# configure the production run (if it hasn't been already)
if not self._productionRunConfigurator:
self.configure(workflowVerbosity)
# make sure the configuration was successful.
if not self._workflowManagers:
raise ConfigurationError("Failed to obtain workflowManagers from configurator")
if not skipConfigCheck:
self.checkConfiguration(checkCare)
# TODO - Re-add when Provenance is complete
# provSetup = self._productionRunConfigurator.getProvenanceSetup()
#
# provSetup.recordProduction()
for workflow in self._workflowManagers["__order"]:
mgr = self._workflowManagers[workflow.getName()]
statusListener = StatusListener()
# this will block until the monitor is created.
monitor = mgr.runWorkflow(statusListener)
self._workflowMonitors.append(monitor)
finally:
self._locked.release()
self._startServiceThread()
print("Production launched.")
print("Waiting for shutdown request.")
def isRunning(self):
"""Determine whether production is currently running
Returns
-------
running : `bool`
Returns True if production is running, otherwise returns False
"""
#
# check each monitor. If any of them are still running,
# the production is still running.
#
for monitor in self._workflowMonitors:
if monitor.isRunning():
return True
with self._locked:
self._locked.running = False
return False
def isDone(self):
"""Determine whether production has completed
Returns
-------
done : `bool`
Returns True if production has completed, otherwise returns False
"""
return self._locked.done
def isRunnable(self):
"""Determine whether production can be run
Returns
-------
runnable : `bool`
Returns True if production can be run, otherwise returns False
Notes
-----
Production is runnable if it isn't already running, or hasn't already been completed. It
can not be re-started once it's already running, or re-run if it has been completed.
"""
return not self.isRunning() and not self.isDone()
def createConfigurator(self, runid, configFile):
"""Create the ProductionRunConfigurator specified in the config file
Parameters
----------
runid : `str`
run id
configFile: `Config`
Config file containing which ProductinRunConfigurator to create
Returns
-------
Initialized ProductionRunConfigurator of the type specified in configFile
"""
log.debug("ProductionRunManager:createConfigurator")
configuratorClass = ProductionRunConfigurator
configuratorClassName = None
if self.config.configurationClass is not None:
configuratorClassName = self.config.configurationClass
if configuratorClassName is not None:
classFactory = NamedClassFactory()
configuratorClass = classFactory.createClass(configuratorClassName)
return configuratorClass(runid, configFile, self.repository)
def checkConfiguration(self, care=1, issueExc=None):
"""Check the configuration of the production
Parameters
----------
care : `int`, optional
The level of "care" to take in checking the configuration.
issueExc : `MultiIssueConfigurationError`, optional
An exception to add addition problems to. (see note)
Notes
-----
In general, the higher the care number, the more checks that are made.
If issueExc is not None, this method will not raise an exception when problems are
encountered; they will be added to the issueExc instance. It is assumed that the caller
will raise that exception as necessary.
"""
log.debug("checkConfiguration")
if not self._workflowManagers:
msg = "%s: production has not been configured yet" % self.runid
if self._name:
msg = "%s %s" % (self._name, msg)
if issueExc is None:
raise ConfigurationError(msg)
else:
issueExc.addProblem(msg)
return
myProblems = issueExc
if myProblems is None:
myProblems = MultiIssueConfigurationError("problems encountered while checking configuration")
# check production-wide configuration
self._productionRunConfigurator.checkConfiguration(care, myProblems)
# check configuration for each workflow
for workflow in self._workflowManagers["__order"]:
workflowMgr = self._workflowManagers[workflow]
workflowMgr.checkConfiguration(care, myProblems)
if not issueExc and myProblems.hasProblems():
raise myProblems
def stopProduction(self, urgency, timeout=1800):
"""Stops all workflows in this production run
Parameters
----------
urgency : `int`
An indicator of how urgently to carry out the shutdown.
timeout : `int`
An time to wait (in sec nds) for workflows to finish.
Returns
-------
success : `bool`
True on successful shutdown of workflows, False otherwise.
Notes
-----
For urgency, it is intended that recognized values should be:
FINISH_PENDING_DATA - end after all currently available data has been processed
END_ITERATION - end after the current data looping iteration
CHECKPOINT - end at next checkpoint opportunity (typically between stages)
NOW - end as soon as possible, foregoing any check-pointing
"""
if not self.isRunning():
log.info("shutdown requested when production is not running")
return
log.info("Shutting down production (urgency=%s)" % urgency)
for workflow in self._workflowManagers["__order"]:
workflowMgr = self._workflowManagers[workflow.getName()]
workflowMgr.stopWorkflow(urgency)
pollintv = 0.2
running = self.isRunning()
lasttime = time.time()
while running and timeout > 0:
time.sleep(pollintv)
for workflow in self._workflowManagers["__order"]:
running = self._workflowManagers[workflow.getName()].isRunning()
if running:
break
timeout -= time.time() - lasttime
if not running:
with self._locked:
self._locked.running = False
self._locked.done = True
else:
log.debug("Failed to shutdown pipelines within timeout: %ss" % timeout)
return False
return True
def getWorkflowNames(self):
"""Accessor to return the "short" name for each workflow in this production.
Returns
-------
names : [ 'wfShortName1', 'wfShortName2' ]
list of "short" names for these workflows.
Notes
-----
These names may have been adjusted to ensure a unique list. These are names that can be
passed by getWorkflowManager(). "Short" names are aliases to the workflows.
"""
if self._workflowManagers:
return self._workflowManagers["__order"]
elif self._productionRunConfigurator:
return self._productionRunConfigurator.getWorkflowNames()
else:
cfg = self.createConfigurator(self.fullConfigFilePath)
return cfg.getWorkflowNames()
def getWorkflowManager(self, name):
"""Accessor to return the named WorkflowManager
Parameters
----------
name : `str`
The name of the WorkflowManager to retrieve
Returns
-------
wfMgr : `WorkflowManager`
A WorkflowManager instance or None if it has not been created yet or name is not one of
the names returned by getWorkflowNames()
"""
if not self._workflowManagers or name not in self._workflowManagers:
return None
return self._workflowManagers[name]
class ThreadedServer(ThreadingMixIn, HTTPServer):
""" threaded server """
def server_bind(self):
HTTPServer.server_bind(self)
def setManager(self, manager):
self.manager = manager
self.socket.settimeout(1)
def serve(self):
while self.manager.isRunning():
self.handle_request()
class _ServiceEndpoint(threading.Thread):
"""This thread deals with incoming requests, and if one is received during production, we
shut everything down.
Parameters
----------
parent : `Thread`
The parent Thread of this Thread.
runid : `str`
run id
pollingIntv : `float`
the polling interval to sleep, in seconds.
listenTimeout : `int`
the interval, in seconds, to wait for an incoming request
Notes
-----
This is a private class.
"""
def __init__(self, parent, runid, pollingIntv=1.0, listenTimeout=10):
threading.Thread.__init__(self)
self.setDaemon(True)
self._runid = runid
self._parent = parent
self._pollintv = pollingIntv
self._timeout = listenTimeout
handlerClass = MakeServiceHandlerClass(parent, runid)
self.server = parent.ThreadedServer(('0.0.0.0', 0), handlerClass)
self.server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
print('server socket listening at %d' % self.server.server_port)
def run(self):
"""Set Manager and serve requests until complete.
"""
self.server.setManager(self._parent)
self.server.serve()
log.debug("Everything shutdown - All finished")
def _startServiceThread(self):
"""Create a shutdown thread, and start it
"""
self._sdthread = ProductionRunManager._ServiceEndpoint(self, self.runid)
self._sdthread.start()
def getShutdownThread(self):
"""Accessor to return shutdown thread for this production
Returns
-------
t : `Thread`
The shutdown Thread.
"""
return self._sdthread
def joinShutdownThread(self):
"""Thread join the shutdown thread for this production
"""
if self._sdthread is not None:
self._sdthread.join()
class ProductionRunManager:
##
# @brief initialize
# @param runid name of the run
# @param configFileName production run config file
# @param repository the config repository to assume; this will
# override the value in the config file
#
def __init__(self, runid, configFileName, repository=None):
# _locked: a container for data to be shared across threads that
# have access to this object.
self._locked = SharedData(False,
{"running": False, "done": False})
## the run id for this production
self.runid = runid
# once the workflows that make up this production is created we will
# cache them here
self._workflowManagers = None
# a list of workflow Monitors
self._workflowMonitors = []
# a list of logger managers
self._loggerManagers = []
# the cached ProductionRunConfigurator instance
self._productionRunConfigurator = None
## the full path the configuration
self.fullConfigFilePath = ""
if os.path.isabs(configFileName) == True:
self.fullConfigFilePath = configFileName
else:
self.fullConfigFilePath = os.path.join(os.path.realpath('.'), configFileName)
## create Production configuration
self.config = ProductionConfig()
# load the production config object
self.config.load(self.fullConfigFilePath)
## the repository location
self.repository = repository
# determine repository location
if not self.repository:
self.repository = self.config.production.repositoryDirectory
if not self.repository:
self.repository = "."
else:
self.repository = EnvString.resolve(self.repository)
# XXX - Check to see if we need to do this still.
# do a little sanity checking on the repository before we continue.
#if not os.path.exists(self.repository):
# raise RuntimeError("specified repository " + self.repository + ": directory not found");
#if not os.path.isdir(self.repository):
# raise RuntimeError("specified repository "+ self.repository + ": not a directory");
# shutdown thread
self._sdthread = None
##
# @brief returns the runid of this production run
#
def getRunId(self):
return self.runid
##
# @brief setup this production to run.
#
# If the production was already configured, it will not be
# reconfigured.
#
# @param workflowVerbosity the verbosity to pass down to configured
# workflows and the pipelines they run.
# @throws ConfigurationError raised if any error arises during configuration or
# while checking the configuration.
#
def configure(self, workflowVerbosity=None):
if self._productionRunConfigurator:
log.info("production has already been configured.")
return
# lock this branch of code
try:
self._locked.acquire()
# TODO - SRP
self._productionRunConfigurator = self.createConfigurator(self.runid,
self.fullConfigFilePath)
workflowManagers = self._productionRunConfigurator.configure(workflowVerbosity)
self._workflowManagers = { "__order": [] }
for wfm in workflowManagers:
self._workflowManagers["__order"].append(wfm)
self._workflowManagers[wfm.getName()] = wfm
loggerManagers = self._productionRunConfigurator.getLoggerManagers()
for lm in loggerManagers:
self._loggerManagers.append(lm)
finally:
self._locked.release()
##
# @brief run the entire production
# @param skipConfigCheck skip the checks that ensures that configuration
# was completed correctly.
# @param workflowVerbosity if not None, override the config-specified logger
# verbosity. This is only used if the run has not
# already been configured via configure().
# @return bool False is returned if the production was already started
# once
# @throws ConfigurationError raised if any error arises during configuration or
# while checking the configuration.
def runProduction(self, skipConfigCheck=False, workflowVerbosity=None):
log.debug("Running production: " + self.runid)
if not self.isRunnable():
if self.isRunning():
log.info("Production Run %s is already running" % self.runid)
if self.isDone():
log.info("Production Run %s has already run; start with new runid" % self.runid)
return False
# set configuration check care level.
# Note: this is not a sanctioned pattern; should be replaced with use
# of default config.
checkCare = 1
if self.config.production.configCheckCare != 0:
checkCare = self.config.production.configCheckCare
if checkCare < 0:
skipConfigCheck = True
# lock this branch of code
try:
self._locked.acquire()
self._locked.running = True
# configure the production run (if it hasn't been already)
if not self._productionRunConfigurator:
self.configure(workflowVerbosity)
# make sure the configuration was successful.
if not self._workflowManagers:
raise ConfigurationError("Failed to obtain workflowManagers from configurator")
if skipConfigCheck == False:
self.checkConfiguration(checkCare)
# launch the logger daemon
for lm in self._loggerManagers:
lm.start()
# TODO - Re-add when Provenance is complete
#provSetup = self._productionRunConfigurator.getProvenanceSetup()
##
#provSetup.recordProduction()
for workflow in self._workflowManagers["__order"]:
mgr = self._workflowManagers[workflow.getName()]
statusListener = StatusListener()
# this will block until the monitor is created.
monitor = mgr.runWorkflow(statusListener, self._loggerManagers)
self._workflowMonitors.append(monitor)
finally:
self._locked.release()
# start the thread that will listen for shutdown events
if self.config.production.productionShutdownTopic != None:
self._startShutdownThread()
# announce data, if it's available
#print "waiting for startup"
#time.sleep(5)
#for workflow in self._workflowManagers["__order"]:
# mgr = self._workflowManagers[workflow.getName()]
# print "mgr = ",mgr
# mgr.announceData()
print "Production launched."
print "Waiting for shutdown request."
##
# @brief determine whether production is currently running
#
def isRunning(self):
#
# check each monitor. If any of them are still running,
# the production is still running.
#
for monitor in self._workflowMonitors:
if monitor.isRunning() == True:
return True
with self._locked:
self._locked.running = False
return False
##
# @brief determine whether production has completed
#
def isDone(self):
return self._locked.done
##
# @brief determine whether production can be run
#
def isRunnable(self):
return not self.isRunning() and not self.isDone()
##
# @brief setup and create the ProductionRunConfigurator
# @return ProductionRunConfigurator
#
#
def createConfigurator(self, runid, configFile):
log.debug("ProductionRunManager:createConfigurator")
configuratorClass = ProductionRunConfigurator
configuratorClassName = None
if self.config.configurationClass != None:
configuratorClassname = self.config.configurationClass
if configuratorClassName != None:
classFactory = NamedClassFactory()
configuratorClass = classFactory.createClass(configuratorClassName)
return configuratorClass(runid, configFile, self.repository)
##
# @brief
# @param care the thoroughness of the checks.
# @param issueExc an instance of MultiIssueConfigurationError to add
# problems to. If not None, this function will not
# raise an exception when problems are encountered; they
# will merely be added to the instance. It is assumed
# that the caller will raise that exception is necessary.
def checkConfiguration(self, care=1, issueExc=None):
# care - level of "care" in checking the configuration to take. In
# general, the higher the number, the more checks that are made.
log.debug("checkConfiguration")
if not self._workflowManagers:
msg = "%s: production has not been configured yet" % self.runid
if self._name:
msg = "%s %s" % (self._name, msg)
if issueExc is None:
raise ConfigurationError(msg)
else:
issueExc.addProblem(msg)
return
myProblems = issueExc
if myProblems is None:
myProblems = MultiIssueConfigurationError("problems encountered while checking configuration")
# check production-wide configuration
self._productionRunConfigurator.checkConfiguration(care, myProblems)
# check configuration for each workflow
for workflow in self._workflowManagers["__order"]:
workflowMgr = self._workflowManagers[workflow]
workflowMgr.checkConfiguration(care, myProblems)
if not issueExc and myProblems.hasProblems():
raise myProblems
##
# @brief stops all workflows in this production run
#
def stopProduction(self, urgency, timeout=1800):
# urgency - an indicator of how urgently to carry out the shutdown.
#
# Recognized values are:
# FINISH_PENDING_DATA - end after all currently available data has
# been processed
# END_ITERATION - end after the current data ooping iteration
# CHECKPOINT - end at next checkpoint opportunity
# (typically between stages)
# NOW - end as soon as possible, forgoing any
# checkpointing
if not self.isRunning():
log.info("shutdown requested when production is not running")
return
log.info("Shutting down production (urgency=%s)" % urgency)
for workflow in self._workflowManagers["__order"]:
workflowMgr = self._workflowManagers[workflow.getName()]
workflowMgr.stopWorkflow(urgency)
pollintv = 0.2
running = self.isRunning()
lasttime = time.time()
while running and timeout > 0:
time.sleep(pollintv)
for workflow in self._workflowManagers["__order"]:
running = self._workflowManagers[workflow.getName()].isRunning()
if running: break
timeout -= time.time() - lasttime
if not running:
with self._locked:
self._locked.running = False
self._locked.done = True
else:
log.debug("Failed to shutdown pipelines within timeout: %ss" % timeout)
return False
# stop loggers after everything else has died
for lm in self._loggerManagers:
lm.stop()
return True
##
# @brief return the "short" name for each workflow in this
# production.
#
# These may have been tweaked to ensure a unique list. These are names
# that can be passed to getWorkflowManager()
#
def getWorkflowNames(self):
if self._workflowManagers:
return self._workflowManagers["__order"]
elif self._productionRunConfigurator:
return self._productionRunConfigurator.getWorkflowNames()
else:
cfg = self.createConfigurator(self.fullConfigFilePath)
return cfg.getWorkflowNames()
##
# @brief return the workflow manager for the given named workflow
# @return a WorkflowManager instance or None if it has not been
# created yet or name is not one of the names returned
# by getWorkflowNames()
def getWorkflowManager(self, name):
if not self._workflowManagers or not self._workflowManagers.has_key(name):
return None
return self._workflowManagers[name]
## shutdown thread
class _ShutdownThread(threading.Thread):
## initialize the shutdown thread
def __init__(self, parent, runid, pollingIntv=0.2, listenTimeout=10):
threading.Thread.__init__(self)
self.setDaemon(True)
self._runid = runid
self._parent = parent
self._pollintv = pollingIntv
self._timeout = listenTimeout
brokerhost = parent.config.production.eventBrokerHost
self._topic = parent.config.production.productionShutdownTopic
self._evsys = events.EventSystem.getDefaultEventSystem()
selector = "RUNID = '%s'" % self._runid
self._evsys.createReceiver(brokerhost, self._topic, selector)
## listen for the shutdown event at regular intervals, and shutdown
# when the event is received.
def run(self):
log.debug("listening for shutdown event at %s s intervals" % self._pollintv)
log.debug("checking for shutdown event")
log.debug("self._timeout = %s" % self._timeout)
shutdownEvent = self._evsys.receiveEvent(self._topic, self._timeout)
while self._parent.isRunning() and shutdownEvent is None:
time.sleep(self._pollintv)
shutdownEvent = self._evsys.receiveEvent(self._topic, self._timeout)
#time.sleep(1)
#shutdownData = self._evsys.receiveEvent(self._topic, 10)
log.debug("DONE!")
if shutdownEvent:
shutdownData = shutdownEvent.getPropertySet()
self._parent.stopProduction(shutdownData.getInt("level"))
log.debug("Everything shutdown - All finished")
def _startShutdownThread(self):
self._sdthread = ProductionRunManager._ShutdownThread(self, self.runid)
self._sdthread.start()
##
# @returns the shutdown thread for this production
def getShutdownThread(self):
return self._sdthread
##
# thread join the shutdown thread for this production
def joinShutdownThread(self):
if self._sdthread is not None:
self._sdthread.join()
class ProductionRunConfigurator(object):
"""Create a basic production run.
Parameters
----------
runid : `str`
run id
configFile : `str`
production configuration file
repository : `str`, optional
file system location of the repository
workflowVerbosity : `int`, optional
verbosity level of the workflow
"""
def __init__(self, runid, configFile, repository=None, workflowVerbosity=None):
log.debug("ProductionRunConfigurator:__init__")
# run id for this production
self.runid = runid
self._prodConfigFile = configFile
# production configuration
self.prodConfig = ProductionConfig()
self.prodConfig.load(configFile)
# location of the repository
self.repository = repository
# verbosity level for this workflow
self.workflowVerbosity = workflowVerbosity
self._provSetup = None
# provenance dictionary
self.provenanceDict = {}
self._wfnames = None
# cache the database configurators for checking the configuraiton.
self._databaseConfigurators = []
# these are config settings which can be overriden from what they
# are in the workflow policies.
# dictionary of configuration override values
self.configOverrides = dict()
production = self.prodConfig.production
if production.logThreshold is not None:
self.configOverrides["execute.logThreshold"] = production.logThreshold
def createWorkflowManager(self, prodConfig, wfName, wfConfig):
"""Create the WorkflowManager for the pipeline with the given shortName
Parameters
----------
prodConfig : `Config`
wfName : `str`
wfConfig : `Config`
"""
log.debug("ProductionRunConfigurator:createWorkflowManager")
wfManager = WorkflowManager(wfName, self.runid, self.repository, prodConfig, wfConfig)
return wfManager
def getProvenanceSetup(self):
"""Accessor to provenance setup information
Returns
-------
s : `str`
provenance setup information
"""
return self._provSetup
def configure(self, workflowVerbosity):
"""Configure this production run
Parameters
----------
workflowVerbosity : `int`
verbosity level of the workflows
Returns
-------
mgrs : [ wfMgr1, wfMgr2 ]
list of workflow managers, one per workflow
"""
log.debug("ProductionRunConfigurator:configure")
# TODO - IMPORTANT - NEXT TWO LINES ARE FOR PROVENANCE
# --------------
# self._provSetup = ProvenanceSetup()
# self._provSetup.addAllProductionConfigFiles(self._prodConfigFile, self.repository)
# --------------
#
# setup the database for each database listed in production config.
# cache the configurators in case we want to check the configuration
# later.
#
databaseConfigs = self.prodConfig.database
for databaseName in databaseConfigs:
databaseConfig = databaseConfigs[databaseName]
cfg = self.createDatabaseConfigurator(databaseConfig)
cfg.setup(self._provSetup)
self._databaseConfigurators.append(cfg)
#
# do specialized production level configuration, if it exists
#
if self.prodConfig.production.configuration.configurationClass is not None:
specialConfigurationConfig = self.prodConfig.production.configuration
# XXX - specialConfigurationConfig maybe?
self.specializedConfigure(specialConfigurationConfig)
workflowConfigs = self.prodConfig.workflow
workflowManagers = []
for wfName in workflowConfigs:
wfConfig = workflowConfigs[wfName]
# copy in appropriate production level info into workflow Node -- ?
workflowManager = self.createWorkflowManager(self.prodConfig, wfName, wfConfig)
workflowLauncher = workflowManager.configure(self._provSetup, workflowVerbosity)
if workflowLauncher is None:
raise MultiIssueConfigurationError("error configuring workflowLauncher")
workflowManagers.append(workflowManager)
return workflowManagers
def checkConfiguration(self, care=1, issueExc=None):
"""Carry out production-wide configuration checks.
Parameters
----------
care : `int`
throughness level of the checks
issueExc : `MultiIssueConfigurationError`
an instance of MultiIssueConfigurationError to add problems to
Raises
------
`MultiIssueConfigurationError`
If issueExc is None, and a configuration error is detected.
Notes
-----
If issueExc is not None, this method will not raise an exception when problems are encountered;
they will merely be added to the instance. It is assumed that the caller will raise the
exception as necessary.
"""
log.debug("checkConfiguration")
myProblems = issueExc
if myProblems is None:
myProblems = MultiIssueConfigurationError("problems encountered while checking configuration")
for dbconfig in self._databaseConfigurators:
print("-> dbconfig = ", dbconfig)
dbconfig.checkConfiguration(care, issueExc)
if not issueExc and myProblems.hasProblems():
raise myProblems
def createDatabaseConfigurator(self, databaseConfig):
"""Create the configurator for database operations
Parameters
----------
databaseConfig: `Config`
database Config object
Returns
-------
configurator : `DatabaseConfigurator`
the configurator specified in the database Config object
"""
log.debug("ProductionRunConfigurator:createDatabaseConfigurator")
className = databaseConfig.configurationClass
classFactory = NamedClassFactory()
configurationClass = classFactory.createClass(className)
configurator = configurationClass(self.runid, databaseConfig, self.prodConfig)
return configurator
def _specializedConfigure(self, specialConfigurationConfig):
"""Do any production-wide setup not covered by the setup of the # databases or the individual
workflows.
Parameters
----------
specialConfigurationConfig : `Config`
Config object for specialized configurations
Notes
-----
This implementation does nothing. Subclasses may override this method
to provide specialized production-wide setup.
"""
pass
def getWorkflowNames(self):
"""Accessor to return workflow names
Returns
-------
names : [ 'wfName1', 'wfName2' ]
list of strings with named workflows
"""
return self.prodConfig.workflowNames
class ProductionRunManager(object):
"""In charge of launching, monitoring, managing, and stopping a production run
Parameters
----------
runid : `str`
name of the run
configFileName : `Config`
production run config file
repository : `str`, optional
the config repository to assume; this will override the value in the config file
"""
def __init__(self, runid, configFileName, repository=None):
# _locked: a container for data to be shared across threads that
# have access to this object.
self._locked = SharedData.SharedData(False, {"running": False, "done": False})
# the run id for this production
self.runid = runid
# once the workflows that make up this production is created we will
# cache them here
self._workflowManagers = None
# a list of workflow Monitors
self._workflowMonitors = []
# the cached ProductionRunConfigurator instance
self._productionRunConfigurator = None
# the full path the configuration
self.fullConfigFilePath = ""
if os.path.isabs(configFileName):
self.fullConfigFilePath = configFileName
else:
self.fullConfigFilePath = os.path.join(os.path.realpath('.'), configFileName)
# create Production configuration
self.config = ProductionConfig()
# load the production config object
self.config.load(self.fullConfigFilePath)
# the repository location
self.repository = repository
# determine repository location
if not self.repository:
self.repository = self.config.production.repositoryDirectory
if not self.repository:
self.repository = "."
else:
self.repository = EnvString.resolve(self.repository)
# shutdown thread
self._sdthread = None
def getRunId(self):
"""Accessor to return the run id for this production run
Returns
-------
The runid of this production run
"""
return self.runid
def configure(self, workflowVerbosity=None):
"""Configure this production run
Parameters
----------
workflowVerbosity : `int`
The verbosity to pass down to configured workflows and the pipelines they run.
Raises
------
`ConfigurationError`
If any error arises during configuration or while checking the configuration.
Notes
-----
If the production was already configured, this call will be ignored and will not be reconfigured.
"""
if self._productionRunConfigurator:
log.info("production has already been configured.")
return
# lock this branch of code
try:
self._locked.acquire()
# TODO - SRP
self._productionRunConfigurator = self.createConfigurator(self.runid,
self.fullConfigFilePath)
workflowManagers = self._productionRunConfigurator.configure(workflowVerbosity)
self._workflowManagers = {"__order": []}
for wfm in workflowManagers:
self._workflowManagers["__order"].append(wfm)
self._workflowManagers[wfm.getName()] = wfm
finally:
self._locked.release()
def runProduction(self, skipConfigCheck=False, workflowVerbosity=None):
"""Run the entire production
Parameters
----------
skipConfigCheck : `bool`
Skips configuration checks, if True
workflowVerbosity: `int`, optional
overrides the config-specified logger verbosity
Raises
------
`ConfigurationError`
if any error arises during configuration or while checking the configuration.
Notes
-----
The skipConfigCheck parameter will be overridden by configCheckCare config parameter, if it exists.
The workflowVerbosity parameter will only be used if the run has not already been configured via
configure().
"""
log.debug("Running production: %s", self.runid)
if not self.isRunnable():
if self.isRunning():
log.info("Production Run %s is already running" % self.runid)
if self.isDone():
log.info("Production Run %s has already run; start with new runid" % self.runid)
return False
# set configuration check care level.
# Note: this is not a sanctioned pattern; should be replaced with use
# of default config.
checkCare = 1
if self.config.production.configCheckCare != 0:
checkCare = self.config.production.configCheckCare
if checkCare < 0:
skipConfigCheck = True
# lock this branch of code
try:
self._locked.acquire()
self._locked.running = True
# configure the production run (if it hasn't been already)
if not self._productionRunConfigurator:
self.configure(workflowVerbosity)
# make sure the configuration was successful.
if not self._workflowManagers:
raise ConfigurationError("Failed to obtain workflowManagers from configurator")
if not skipConfigCheck:
self.checkConfiguration(checkCare)
# TODO - Re-add when Provenance is complete
# provSetup = self._productionRunConfigurator.getProvenanceSetup()
#
# provSetup.recordProduction()
for workflow in self._workflowManagers["__order"]:
mgr = self._workflowManagers[workflow.getName()]
statusListener = StatusListener()
# this will block until the monitor is created.
monitor = mgr.runWorkflow(statusListener)
self._workflowMonitors.append(monitor)
finally:
self._locked.release()
self._startServiceThread()
print("Production launched.")
print("Waiting for shutdown request.")
def isRunning(self):
"""Determine whether production is currently running
Returns
-------
running : `bool`
Returns True if production is running, otherwise returns False
"""
#
# check each monitor. If any of them are still running,
# the production is still running.
#
for monitor in self._workflowMonitors:
if monitor.isRunning():
return True
with self._locked:
self._locked.running = False
return False
def isDone(self):
"""Determine whether production has completed
Returns
-------
done : `bool`
Returns True if production has completed, otherwise returns False
"""
return self._locked.done
def isRunnable(self):
"""Determine whether production can be run
Returns
-------
runnable : `bool`
Returns True if production can be run, otherwise returns False
Notes
-----
Production is runnable if it isn't already running, or hasn't already been completed. It
can not be re-started once it's already running, or re-run if it has been completed.
"""
return not self.isRunning() and not self.isDone()
def createConfigurator(self, runid, configFile):
"""Create the ProductionRunConfigurator specified in the config file
Parameters
----------
runid : `str`
run id
configFile: `Config`
Config file containing which ProductinRunConfigurator to create
Returns
-------
Initialized ProductionRunConfigurator of the type specified in configFile
"""
log.debug("ProductionRunManager:createConfigurator")
configuratorClass = ProductionRunConfigurator
configuratorClassName = None
if self.config.configurationClass is not None:
configuratorClassName = self.config.configurationClass
if configuratorClassName is not None:
classFactory = NamedClassFactory()
configuratorClass = classFactory.createClass(configuratorClassName)
return configuratorClass(runid, configFile, self.repository)
def checkConfiguration(self, care=1, issueExc=None):
"""Check the configuration of the production
Parameters
----------
care : `int`, optional
The level of "care" to take in checking the configuration.
issueExc : `MultiIssueConfigurationError`, optional
An exception to add addition problems to. (see note)
Notes
-----
In general, the higher the care number, the more checks that are made.
If issueExc is not None, this method will not raise an exception when problems are
encountered; they will be added to the issueExc instance. It is assumed that the caller
will raise that exception as necessary.
"""
log.debug("checkConfiguration")
if not self._workflowManagers:
msg = "%s: production has not been configured yet" % self.runid
if self._name:
msg = "%s %s" % (self._name, msg)
if issueExc is None:
raise ConfigurationError(msg)
else:
issueExc.addProblem(msg)
return
myProblems = issueExc
if myProblems is None:
myProblems = MultiIssueConfigurationError("problems encountered while checking configuration")
# check production-wide configuration
self._productionRunConfigurator.checkConfiguration(care, myProblems)
# check configuration for each workflow
for workflow in self._workflowManagers["__order"]:
workflowMgr = self._workflowManagers[workflow]
workflowMgr.checkConfiguration(care, myProblems)
if not issueExc and myProblems.hasProblems():
raise myProblems
def stopProduction(self, urgency, timeout=1800):
"""Stops all workflows in this production run
Parameters
----------
urgency : `int`
An indicator of how urgently to carry out the shutdown.
timeout : `int`
An time to wait (in sec nds) for workflows to finish.
Returns
-------
success : `bool`
True on successful shutdown of workflows, False otherwise.
Notes
-----
For urgency, it is intended that recognized values should be:
FINISH_PENDING_DATA - end after all currently available data has been processed
END_ITERATION - end after the current data looping iteration
CHECKPOINT - end at next checkpoint opportunity (typically between stages)
NOW - end as soon as possible, foregoing any check-pointing
"""
if not self.isRunning():
log.info("shutdown requested when production is not running")
return
log.info("Shutting down production (urgency=%s)" % urgency)
for workflow in self._workflowManagers["__order"]:
workflowMgr = self._workflowManagers[workflow.getName()]
workflowMgr.stopWorkflow(urgency)
pollintv = 0.2
running = self.isRunning()
lasttime = time.time()
while running and timeout > 0:
time.sleep(pollintv)
for workflow in self._workflowManagers["__order"]:
running = self._workflowManagers[workflow.getName()].isRunning()
if running:
break
timeout -= time.time() - lasttime
if not running:
with self._locked:
self._locked.running = False
self._locked.done = True
else:
log.debug("Failed to shutdown pipelines within timeout: %ss" % timeout)
return False
return True
def getWorkflowNames(self):
"""Accessor to return the "short" name for each workflow in this production.
Returns
-------
names : [ 'wfShortName1', 'wfShortName2' ]
list of "short" names for these workflows.
Notes
-----
These names may have been adjusted to ensure a unique list. These are names that can be
passed by getWorkflowManager(). "Short" names are aliases to the workflows.
"""
if self._workflowManagers:
return self._workflowManagers["__order"]
elif self._productionRunConfigurator:
return self._productionRunConfigurator.getWorkflowNames()
else:
cfg = self.createConfigurator(self.fullConfigFilePath)
return cfg.getWorkflowNames()
def getWorkflowManager(self, name):
"""Accessor to return the named WorkflowManager
Parameters
----------
name : `str`
The name of the WorkflowManager to retrieve
Returns
-------
wfMgr : `WorkflowManager`
A WorkflowManager instance or None if it has not been created yet or name is not one of
the names returned by getWorkflowNames()
"""
if not self._workflowManagers or name not in self._workflowManagers:
return None
return self._workflowManagers[name]
class ThreadedServer(ThreadingMixIn, HTTPServer):
""" threaded server """
def server_bind(self):
HTTPServer.server_bind(self)
def setManager(self, manager):
self.manager = manager
self.socket.settimeout(1)
def serve(self):
while self.manager.isRunning():
self.handle_request()
class _ServiceEndpoint(threading.Thread):
"""This thread deals with incoming requests, and if one is received during production, we
shut everything down.
Parameters
----------
parent : `Thread`
The parent Thread of this Thread.
runid : `str`
run id
pollingIntv : `float`
the polling interval to sleep, in seconds.
listenTimeout : `int`
the interval, in seconds, to wait for an incoming request
Notes
-----
This is a private class.
"""
def __init__(self, parent, runid, pollingIntv=1.0, listenTimeout=10):
threading.Thread.__init__(self)
self.setDaemon(True)
self._runid = runid
self._parent = parent
self._pollintv = pollingIntv
self._timeout = listenTimeout
handlerClass = MakeServiceHandlerClass(parent, runid)
self.server = parent.ThreadedServer(('0.0.0.0', 0), handlerClass)
self.server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
print('server socket listening at %d' % self.server.server_port)
def run(self):
"""Set Manager and serve requests until complete.
"""
self.server.setManager(self._parent)
self.server.serve()
log.debug("Everything shutdown - All finished")
def _startServiceThread(self):
"""Create a shutdown thread, and start it
"""
self._sdthread = ProductionRunManager._ServiceEndpoint(self, self.runid)
self._sdthread.start()
def getShutdownThread(self):
"""Accessor to return shutdown thread for this production
Returns
-------
t : `Thread`
The shutdown Thread.
"""
return self._sdthread
def joinShutdownThread(self):
"""Thread join the shutdown thread for this production
"""
if self._sdthread is not None:
self._sdthread.join()
class ProductionRunConfigurator:
## initialize
def __init__(self, runid, configFile, repository=None, workflowVerbosity=None):
log.debug("ProductionRunConfigurator:__init__")
## run id for this production
self.runid = runid
self._prodConfigFile = configFile
## production configuration
self.prodConfig = ProductionConfig()
self.prodConfig.load(configFile)
## location of the repostiry
self.repository = repository
## verbosity level for this workflow
self.workflowVerbosity = workflowVerbosity
self._provSetup = None
## provenance dictionary
self.provenanceDict = {}
self._wfnames = None
# cache the database configurators for checking the configuraiton.
self._databaseConfigurators = []
# logger managers
self._loggerManagers = []
## hostname of the event broker
self.eventBrokerHost = None
# these are config settings which can be overriden from what they
# are in the workflow policies.
## dictionary of configuration override values
self.configOverrides = dict()
production = self.prodConfig.production
if production.eventBrokerHost != None:
self.eventBrokerHost = production.eventBrokerHost
self.configOverrides["execute.eventBrokerHost"] = production.eventBrokerHost
if production.logThreshold != None:
self.configOverrides["execute.logThreshold"] = production.logThreshold
if production.productionShutdownTopic != None:
self.configOverrides["execute.shutdownTopic"] = production.productionShutdownTopic
##
# @brief create the WorkflowManager for the pipelien with the given shortName
#
def createWorkflowManager(self, prodConfig, wfName, wfConfig):
log.debug("ProductionRunConfigurator:createWorkflowManager")
wfManager = WorkflowManager(wfName, self.runid, self.repository, prodConfig, wfConfig)
return wfManager
##
# @brief return provenanceSetup
#
def getProvenanceSetup(self):
return self._provSetup
##
# @brief configure this production run
#
def configure(self, workflowVerbosity):
log.debug("ProductionRunConfigurator:configure")
# TODO - IMPORTANT - NEXT TWO LINES ARE FOR PROVENANCE
# --------------
#self._provSetup = ProvenanceSetup()
#self._provSetup.addAllProductionConfigFiles(self._prodConfigFile, self.repository)
# --------------
#
# setup the database for each database listed in production config.
# cache the configurators in case we want to check the configuration
# later.
#
#databaseConfigNames = self.prodConfig.databaseConfigNames
databaseConfigs = self.prodConfig.database
#for databaseName in databaseConfigNames:
for databaseName in databaseConfigs:
databaseConfig = databaseConfigs[databaseName]
cfg = self.createDatabaseConfigurator(databaseConfig)
cfg.setup(self._provSetup)
dbInfo = cfg.getDBInfo()
# check to see if we're supposed to launch a logging daemon
if databaseConfig.logger != None:
loggerConfig = databaseConfig.logger
if loggerConfig.launch != None:
launch = loggerConfig.launch
loggerManager = None
if launch == True:
loggerManager = LoggerManager(self.eventBrokerHost, self.runid, dbInfo["host"], dbInfo["port"], dbInfo["dbrun"])
else:
loggerManager = LoggerManager(self.eventBrokerHost, self.runid)
if loggerManager is not None:
self._loggerManagers.append(loggerManager)
self._databaseConfigurators.append(cfg)
#
# do specialized production level configuration, if it exists
#
if self.prodConfig.production.configuration.configurationClass != None:
specialConfigurationConfig = self.prodConfig.production.configuration
# XXX - specialConfigurationConfig maybe?
self.specializedConfigure(specialConfigurationConfig)
#workflowNames = self.prodConfig.workflowNames
workflowConfigs = self.prodConfig.workflow
workflowManagers = []
for wfName in workflowConfigs:
wfConfig = workflowConfigs[wfName]
# copy in appropriate production level info into workflow Node -- ?
workflowManager = self.createWorkflowManager(self.prodConfig, wfName, wfConfig)
workflowLauncher = workflowManager.configure(self._provSetup, workflowVerbosity)
workflowManagers.append(workflowManager)
return workflowManagers
##
# @return list of logger managers
def getLoggerManagers(self):
return self._loggerManagers
##
# @brief carry out production-wide configuration checks.
# @param care the thoroughness of the checks.
# @param issueExc an instance of MultiIssueConfigurationError to add
# problems to. If not None, this function will not
# raise an exception when problems are encountered; they
# will merely be added to the instance. It is assumed
# that the caller will raise that exception is necessary.
#
def checkConfiguration(self, care=1, issueExc=None):
log.debug("checkConfiguration")
myProblems = issueExc
if myProblems is None:
myProblems = MultiIssueConfigurationError("problems encountered while checking configuration")
for dbconfig in self._databaseConfigurators:
print "-> dbconfig = ",dbconfig
dbconfig.checkConfiguration(care, issueExc)
if not issueExc and myProblems.hasProblems():
raise myProblems
##
# @brief lookup and create the configurator for database operations
#
def createDatabaseConfigurator(self, databaseConfig):
log.debug("ProductionRunConfigurator:createDatabaseConfigurator")
className = databaseConfig.configurationClass
classFactory = NamedClassFactory()
configurationClass = classFactory.createClass(className)
configurator = configurationClass(self.runid, databaseConfig, self.prodConfig, None)
return configurator
##
# @brief do any production-wide setup not covered by the setup of the
# databases or the individual workflows.
#
# This implementation does nothing. Subclasses may override this method
# to provide specialized production-wide setup.
#
def _specializedConfigure(self, specialConfigurationConfig):
pass
##
# @brief return the workflow names to be used for this set of workflows
#
def getWorkflowNames(self):
return self.prodConfig.workflowNames
class ProductionRunConfigurator(object):
"""Create a basic production run.
Parameters
----------
runid : `str`
run id
configFile : `str`
production configuration file
repository : `str`, optional
file system location of the repository
workflowVerbosity : `int`, optional
verbosity level of the workflow
"""
def __init__(self,
runid,
configFile,
repository=None,
workflowVerbosity=None):
log.debug("ProductionRunConfigurator:__init__")
# run id for this production
self.runid = runid
self._prodConfigFile = configFile
# production configuration
self.prodConfig = ProductionConfig()
self.prodConfig.load(configFile)
# location of the repository
self.repository = repository
# verbosity level for this workflow
self.workflowVerbosity = workflowVerbosity
self._provSetup = None
# provenance dictionary
self.provenanceDict = {}
self._wfnames = None
# cache the database configurators for checking the configuraiton.
self._databaseConfigurators = []
# these are config settings which can be overriden from what they
# are in the workflow policies.
# dictionary of configuration override values
self.configOverrides = dict()
production = self.prodConfig.production
if production.logThreshold is not None:
self.configOverrides[
"execute.logThreshold"] = production.logThreshold
def createWorkflowManager(self, prodConfig, wfName, wfConfig):
"""Create the WorkflowManager for the pipeline with the given shortName
Parameters
----------
prodConfig : `Config`
wfName : `str`
wfConfig : `Config`
"""
log.debug("ProductionRunConfigurator:createWorkflowManager")
wfManager = WorkflowManager(wfName, self.runid, self.repository,
prodConfig, wfConfig)
return wfManager
def getProvenanceSetup(self):
"""Accessor to provenance setup information
Returns
-------
s : `str`
provenance setup information
"""
return self._provSetup
def configure(self, workflowVerbosity):
"""Configure this production run
Parameters
----------
workflowVerbosity : `int`
verbosity level of the workflows
Returns
-------
mgrs : [ wfMgr1, wfMgr2 ]
list of workflow managers, one per workflow
"""
log.debug("ProductionRunConfigurator:configure")
# TODO - IMPORTANT - NEXT TWO LINES ARE FOR PROVENANCE
# --------------
# self._provSetup = ProvenanceSetup()
# self._provSetup.addAllProductionConfigFiles(self._prodConfigFile, self.repository)
# --------------
#
# setup the database for each database listed in production config.
# cache the configurators in case we want to check the configuration
# later.
#
databaseConfigs = self.prodConfig.database
for databaseName in databaseConfigs:
databaseConfig = databaseConfigs[databaseName]
cfg = self.createDatabaseConfigurator(databaseConfig)
cfg.setup(self._provSetup)
self._databaseConfigurators.append(cfg)
#
# do specialized production level configuration, if it exists
#
if self.prodConfig.production.configuration.configurationClass is not None:
specialConfigurationConfig = self.prodConfig.production.configuration
# XXX - specialConfigurationConfig maybe?
self.specializedConfigure(specialConfigurationConfig)
workflowConfigs = self.prodConfig.workflow
workflowManagers = []
for wfName in workflowConfigs:
wfConfig = workflowConfigs[wfName]
# copy in appropriate production level info into workflow Node -- ?
workflowManager = self.createWorkflowManager(
self.prodConfig, wfName, wfConfig)
workflowLauncher = workflowManager.configure(
self._provSetup, workflowVerbosity)
if workflowLauncher is None:
raise MultiIssueConfigurationError(
"error configuring workflowLauncher")
workflowManagers.append(workflowManager)
return workflowManagers
def checkConfiguration(self, care=1, issueExc=None):
"""Carry out production-wide configuration checks.
Parameters
----------
care : `int`
throughness level of the checks
issueExc : `MultiIssueConfigurationError`
an instance of MultiIssueConfigurationError to add problems to
Raises
------
`MultiIssueConfigurationError`
If issueExc is None, and a configuration error is detected.
Notes
-----
If issueExc is not None, this method will not raise an exception when problems are encountered;
they will merely be added to the instance. It is assumed that the caller will raise the
exception as necessary.
"""
log.debug("checkConfiguration")
myProblems = issueExc
if myProblems is None:
myProblems = MultiIssueConfigurationError(
"problems encountered while checking configuration")
for dbconfig in self._databaseConfigurators:
print("-> dbconfig = ", dbconfig)
dbconfig.checkConfiguration(care, issueExc)
if not issueExc and myProblems.hasProblems():
raise myProblems
def createDatabaseConfigurator(self, databaseConfig):
"""Create the configurator for database operations
Parameters
----------
databaseConfig: `Config`
database Config object
Returns
-------
configurator : `DatabaseConfigurator`
the configurator specified in the database Config object
"""
log.debug("ProductionRunConfigurator:createDatabaseConfigurator")
className = databaseConfig.configurationClass
classFactory = NamedClassFactory()
configurationClass = classFactory.createClass(className)
configurator = configurationClass(self.runid, databaseConfig,
self.prodConfig)
return configurator
def _specializedConfigure(self, specialConfigurationConfig):
"""Do any production-wide setup not covered by the setup of the # databases or the individual
workflows.
Parameters
----------
specialConfigurationConfig : `Config`
Config object for specialized configurations
Notes
-----
This implementation does nothing. Subclasses may override this method
to provide specialized production-wide setup.
"""
pass
def getWorkflowNames(self):
"""Accessor to return workflow names
Returns
-------
names : [ 'wfName1', 'wfName2' ]
list of strings with named workflows
"""
return self.prodConfig.workflowNames