def setup(self):
deffile = DefaultPolicyFile("ctrl_sched", "GetAJob_dict.paf",
"policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self, "policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
self.jobid = None
self.tagLogger(None)
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["jobIdentity"] = \
self.policy.getString("outputKeys.jobIdentity")
self.clipboardKeys["inputDatasets"] = \
self.policy.getString("outputKeys.inputDatasets")
self.clipboardKeys["outputDatasets"] = \
self.policy.getString("outputKeys.outputDatasets")
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("outputKeys.completedDatasets")
self.log.log(Log.INFO - 1,
"clipboard keys: " + str(self.clipboardKeys))
topic = self.policy.getString("pipelineEvent")
self.client = GetAJobClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), self.log)
self.log.log(Log.INFO - 1,
"Using OriginatorId = %d" % self.client.getOriginatorId())
def setup(self, policyDict="DataReady_dict.paf"):
deffile = DefaultPolicyFile("ctrl_sched", policyDict, "policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self, "policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("inputKeys.completedDatasets")
self.clipboardKeys["possibleDatasets"] = \
self.policy.getString("inputKeys.possibleDatasets")
self.dataclients = []
clpols = []
if self.policy.exists("datasets"):
clpols = self.policy.getPolicyArray("datasets")
for pol in clpols:
dstype = None
if pol.exists("datasetType"):
dstype = pol.getString("datasetType")
topic = pol.getString("dataReadyEvent")
reportAll = pol.getBool("reportAllPossible")
client = DataReadyClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), dstype,
reportAll)
self.dataclients.append(client)
def setup(self, policyDict="DataReady_dict.paf"):
deffile = DefaultPolicyFile("ctrl_sched", policyDict, "policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self,"policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("inputKeys.completedDatasets")
self.clipboardKeys["possibleDatasets"] = \
self.policy.getString("inputKeys.possibleDatasets")
self.dataclients = []
clpols = []
if self.policy.exists("datasets"):
clpols = self.policy.getPolicyArray("datasets")
for pol in clpols:
dstype = None
if pol.exists("datasetType"):
dstype = pol.getString("datasetType")
topic = pol.getString("dataReadyEvent")
reportAll = pol.getBool("reportAllPossible")
client = DataReadyClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), dstype,
reportAll)
self.dataclients.append(client)
def setup(self):
deffile = DefaultPolicyFile("ctrl_sched","GetAJob_dict.paf","policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self,"policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
self.jobid = None
self.tagLogger(None)
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["jobIdentity"] = \
self.policy.getString("outputKeys.jobIdentity")
self.clipboardKeys["inputDatasets"] = \
self.policy.getString("outputKeys.inputDatasets")
self.clipboardKeys["outputDatasets"] = \
self.policy.getString("outputKeys.outputDatasets")
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("outputKeys.completedDatasets")
self.log.log(Log.INFO-1, "clipboard keys: " + str(self.clipboardKeys))
topic = self.policy.getString("pipelineEvent")
self.client = GetAJobClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), self.log)
self.log.log(Log.INFO-1,
"Using OriginatorId = %d" % self.client.getOriginatorId())
def main():
"""
run the job office
"""
(cl.opts, cl.args) = cl.parse_args()
if len(cl.args) == 0:
fail("Missing policy file")
if not os.path.exists(cl.args[0]):
fail("%s: policy file not found" % cl.args[0])
if not os.path.exists(cl.opts.rootdir):
fail("%s: root directory not found" % cl.opts.rootdir)
if not cl.opts.runid:
logger.log(Log.WARN, "No RunID given (via -r)")
defpolf = DefaultPolicyFile("ctrl_sched", "JobOffice_dict.paf", "policies")
policy = Policy.createPolicy(cl.args[0])
policy.mergeDefaults(Policy.createPolicy(defpolf,
defpolf.getRepositoryPath()))
name = policy.getString("name")
# set job office root directory
if not os.path.isabs(cl.opts.rootdir):
cl.opts.rootdir = os.path.abspath(cl.opts.rootdir)
persistdir = os.path.join(cl.opts.rootdir, name)
if policy.exists("persist.dir"):
persistdir = policy.get("persist.dir") % \
{"schedroot": cl.opts.rootdir, "name": name }
# create the logger(s)
logfile = cl.opts.logfile
if not logfile:
logfile = os.path.join(persistdir, "joboffice.log")
if not os.path.exists(logfile):
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
if not cl.opts.asdaemon or cl.opts.toscreen:
ofclogger = DualLog(logfile, Log.DEBUG, Log.DEBUG, False)
# logging bug workaround
ofclogger.setScreenVerbose(False)
else:
ofclogger = Log()
ofclogger.addDestination(logfile)
ofclogger.setThreshold(run.verbosity2threshold(cl.opts.logverb, 0))
ofclogger.log(-2,"office threshold: %i" % ofclogger.getThreshold())
try:
# create the JobOffice instance
office = createJobOffice(cl.opts.rootdir, policy, ofclogger,
cl.opts.runid, cl.opts.brokerhost,
cl.opts.brokerport)
except Exception, ex:
logger.log(Log.FATAL, "Failed to create job office: " + str(ex))
raise
sys.exit(1)
def __init__(self, blackboard, policy, logger=None):
"""
create the scheduler
@param blackboard the blackboard that this scheduler will update.
@param policy the "schedule" policy used to configure this
scheduler
@param logger a log to use for messages
"""
Scheduler.__init__(self, blackboard, logger, True)
defpol = DefaultPolicyFile("ctrl_sched",
"DataTriggeredScheduler_dict.paf",
"policies")
defpol = Policy.createPolicy(defpol, defpol.getRepositoryPath(), False)
policy.mergeDefaults(defpol)
self.triggers = []
trigps = policy.getArray("trigger")
for trigp in trigps:
self.triggers.append(Trigger.fromPolicy(trigp))
self.inputdata = []
inpps = policy.getArray("job.input")
for dsp in inpps:
self.inputdata.append(Trigger.fromPolicy(dsp, True))
self.outputdata = []
outpps = policy.getArray("job.output")
for dsp in outpps:
self.outputdata.append(Trigger.fromPolicy(dsp, True))
self.jobIdConf = None
if policy.exists("job.identity"):
self.jobIdConf = policy.getPolicy("job.identity")
self.nametmpl = None
pol = policy.get("job.name")
self.defaultName = pol.getString("default")
if pol.exists("template"):
self.nametmpl = pol.getString("template")
self.nameNumber = pol.getInt("initCounter")
self.jobRetries = None
if policy.exists("job.retries"):
self.jobRetries = policy.getInt("job.retries")
def testSampleCode(self):
policyFile = DefaultPolicyFile("pex_policy", "defaults_dictionary_complete.paf",
"tests/dictionary")
defaults = Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True)
policy = Policy()
policy.mergeDefaults(defaults)
self.assertTrue(policy.canValidate())
policy = Policy()
policy.mergeDefaults(defaults, False)
self.assertFalse(policy.canValidate())
# even if not keeping it around for future validation, validate the merge now
policy = Policy()
policy.set("bad", "worse")
self.assertValidationError(ValidationError.UNKNOWN_NAME,
policy.mergeDefaults, defaults, False)
self.assertFalse(policy.canValidate())
def testSampleCode(self):
policyFile = DefaultPolicyFile("pex_policy", "defaults_dictionary_complete.paf",
"tests/dictionary")
defaults = Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True)
policy = Policy()
policy.mergeDefaults(defaults)
self.assert_(policy.canValidate())
policy = Policy()
policy.mergeDefaults(defaults, False)
self.assert_(not policy.canValidate())
# even if not keeping it around for future validation, validate the merge now
policy = Policy()
policy.set("bad", "worse")
self.assertValidationError(ValidationError.UNKNOWN_NAME,
policy.mergeDefaults, defaults, False)
self.assert_(not policy.canValidate())
def __init__(self,
rootdir,
policy=None,
log=None,
runId=None,
brokerHost=None,
brokerPort=None,
forDaemon=False):
"""
create a JobOffice that is triggered by the appearence of data
that matched filters specified in the policy file.
@param rootdir the root directory where job offices may set up
its blackboard data. This JobOffice will
create a subdirectory here for its data with
a name set by the "name" policy paramter.
@param policy the policy to use to configure this JobOffice
@param defPolicyFile the DefaultPolicyFile to use for defaults. If
this points to a dictionary, a policy
validation will be done. If None, an
internally identified default policy file
will be used.
@param log a logger to use; if None, the default logger will
will be used. A child log will be created.
@param runId the run ID to restrict our attention to. If
not None, incoming events will be restricted
to the given runId.
@param brokerHost the machine where the event broker is running.
If None (default), the host given in the
policy is used. This parameter is for
carrying an override from the command line.
@param brokerHost the port to use to connect to the event broker.
If None (default), the port given in the
policy is used. This parameter is for
carrying an override from the command line.
@param forDaemon if true, the creation of the Event channels will
be delayed until run() is called. This allows
the caller to fork as needed without losing
the connections with the event broker. This
also means that some public functions (other
than run() and start()) will not work
properly until ensureReady() is called.
"""
dpolf = DefaultPolicyFile("ctrl_sched",
"DataTriggeredJobOffice_dict.paf",
"policies")
_BaseJobOffice.__init__(self, rootdir, policy, dpolf, log, runId,
brokerHost, brokerPort, forDaemon, True)
# create a scheduler based on "schedule.className"
self.scheduler = \
DataTriggeredScheduler(self.bb, self.policy.getPolicy("schedule"),
self.log)
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not StringIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "StringIDFilter_dict.paf",
"policies")
StringIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if StringIDFilter._dictionary:
p.mergeDefaults(StringIDFilter._dictionary)
name = "unknown"
vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("value"): vals = policy.getArray("value")
return StringIDFilter(name, vals)
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not IntegerIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "IntegerIDFilter_dict.paf",
"policies")
IntegerIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if IntegerIDFilter._dictionary:
p.mergeDefaults(IntegerIDFilter._dictionary)
name = "unknown"
min = lim = vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("min"): min = policy.getInt("min")
if policy.exists("lim"): lim = policy.getInt("lim")
if policy.exists("value"): vals = policy.getArray("value")
return IntegerIDFilter(name, min, lim, vals)
def __init__(self,
rootdir,
policy=None,
defPolicyFile=None,
log=None,
runId=None,
brokerHost=None,
brokerPort=None,
forDaemon=False,
fromSubclass=False):
"""
create the JobOffice
@param rootdir the root directory where job offices may set up
its blackboard data. This JobOffice will
create a subdirectory here for its data with
a name set by the "name" policy paramter.
@param policy the policy to use to configure this JobOffice
@param defPolicyFile the DefaultPolicyFile to use for defaults. If
this points to a dictionary, a policy
validation will be done. If None, an
internally identified default policy file
will be used.
@param log a logger to use; if None, the default logger will
will be used. A child log will be created.
@param runId the run ID to restrict our attention to. If
not None, incoming events will be restricted
to the given runId.
@param brokerHost the machine where the event broker is running.
If None (default), the host given in the
policy is used. This parameter is for
carrying an override from the command line.
@param brokerPort the port to use to connect to the event broker.
If None (default), the port given in the
policy is used. This parameter is for
carrying an override from the command line.
@param forDaemon if true, the creation of the Event channels will
be delayed until run() is called. This allows
the caller to fork as needed without losing
the connections with the event broker. This
also means that some public functions (other
than run() and start()) will not work
properly until ensureReady() is called.
@param fromSubclass the flag indicating that this constructor is
being properly called. Calls to this
constructor from a subclass constructor should
set this to True.
"""
self._checkAbstract(fromSubclass, "_BasicJobOffice")
# start by establishing policy data
if not defPolicyFile:
defPolicyFile = DefaultPolicyFile("ctrl_sched",
"baseJobOffice_dict.paf",
"policies")
defaults = Policy.createPolicy(defPolicyFile,
defPolicyFile.getRepositoryPath(), True)
if not policy:
policy = Policy()
self.policy = policy
if defaults.canValidate():
self.policy.mergeDefaults(defaults.getDictionary())
else:
self.policy.mergeDefaults(defaults)
# instantiate parent class
name = self.policy.get("name")
persistDir = self.policy.get("persist.dir") % {
"schedroot": rootdir,
"name": name
}
if not os.path.exists(persistDir):
os.makedirs(persistDir)
JobOffice.__init__(self, persistDir, log, runId, True)
self.setName(name)
# logger
self.log = Log(self.log, self.name)
# initialize some data from policy
self.initialWait = self.policy.get("listen.initialWait")
self.emptyWait = self.policy.get("listen.emptyWait")
self.dataTopics = self.policy.getArray("listen.dataReadyEvent")
self.jobTopic = self.policy.get("listen.pipelineEvent")
self.stopTopic = self.policy.get("listen.stopEvent")
self.jobOfficeTopic = self.policy.get("listen.jobOfficeEvent")
self.jobOfficeStatusTopic = self.policy.get("listen.jobOfficeStatus")
self.highWatermark = self.policy.get("listen.highWatermark")
# initialize the event system
self.jobReadyEvRcvr = self.dataEvRcvrs = None
self.jobDoneEvRcvr = self.jobAcceptedEvRcvr = None
self.brokerHost = brokerHost
self.brokerPort = brokerPort
if not self.brokerPort and self.policy.exists("listen.brokerHostPort"):
self.brokerPort = self.policy.get("listen.brokerHostPort")
if not self.brokerHost and (not self.brokerPort
or self.brokerPort > 0):
self.brokerHost = self.policy.get("listen.brokerHostName")
self.dataEvRcvrs = None
self.jobReadyEvRcvr = None
self.jobDoneEvRcvr = None
self.jobAcceptedEvRcvr = None
self.jobAssignEvTrx = None
self.jobOfficeRcvr = None
if not forDaemon:
self._setEventPipes()
def __init__(self, rootdir, policy=None, defPolicyFile=None, log=None,
runId=None, brokerHost=None, brokerPort=None, forDaemon=False,
fromSubclass=False):
"""
create the JobOffice
@param rootdir the root directory where job offices may set up
its blackboard data. This JobOffice will
create a subdirectory here for its data with
a name set by the "name" policy paramter.
@param policy the policy to use to configure this JobOffice
@param defPolicyFile the DefaultPolicyFile to use for defaults. If
this points to a dictionary, a policy
validation will be done. If None, an
internally identified default policy file
will be used.
@param log a logger to use; if None, the default logger will
will be used. A child log will be created.
@param runId the run ID to restrict our attention to. If
not None, incoming events will be restricted
to the given runId.
@param brokerHost the machine where the event broker is running.
If None (default), the host given in the
policy is used. This parameter is for
carrying an override from the command line.
@param brokerPort the port to use to connect to the event broker.
If None (default), the port given in the
policy is used. This parameter is for
carrying an override from the command line.
@param forDaemon if true, the creation of the Event channels will
be delayed until run() is called. This allows
the caller to fork as needed without losing
the connections with the event broker. This
also means that some public functions (other
than run() and start()) will not work
properly until ensureReady() is called.
@param fromSubclass the flag indicating that this constructor is
being properly called. Calls to this
constructor from a subclass constructor should
set this to True.
"""
self._checkAbstract(fromSubclass, "_BasicJobOffice")
# start by establishing policy data
if not defPolicyFile:
defPolicyFile = DefaultPolicyFile("ctrl_sched",
"baseJobOffice_dict.paf",
"policies")
defaults = Policy.createPolicy(defPolicyFile,
defPolicyFile.getRepositoryPath(),
True)
if not policy:
policy = Policy()
self.policy = policy
if defaults.canValidate():
self.policy.mergeDefaults(defaults.getDictionary())
else:
self.policy.mergeDefaults(defaults)
# instantiate parent class
name = self.policy.get("name")
persistDir = self.policy.get("persist.dir") % {"schedroot": rootdir,
"name": name }
if not os.path.exists(persistDir):
os.makedirs(persistDir)
JobOffice.__init__(self, persistDir, log, runId, True)
self.setName(name)
# logger
self.log = Log(self.log, self.name)
# initialize some data from policy
self.initialWait = self.policy.get("listen.initialWait")
self.emptyWait = self.policy.get("listen.emptyWait")
self.dataTopics = self.policy.getArray("listen.dataReadyEvent")
self.jobTopic = self.policy.get("listen.pipelineEvent")
self.stopTopic = self.policy.get("listen.stopEvent")
self.jobOfficeTopic = self.policy.get("listen.jobOfficeEvent")
self.jobOfficeStatusTopic = self.policy.get("listen.jobOfficeStatus")
self.highWatermark = self.policy.get("listen.highWatermark")
# initialize the event system
self.jobReadyEvRcvr = self.dataEvRcvrs = None
self.jobDoneEvRcvr = self.jobAcceptedEvRcvr = None
self.brokerHost = brokerHost
self.brokerPort = brokerPort
if not self.brokerPort and self.policy.exists("listen.brokerHostPort"):
self.brokerPort = self.policy.get("listen.brokerHostPort")
if not self.brokerHost and (not self.brokerPort or self.brokerPort > 0):
self.brokerHost = self.policy.get("listen.brokerHostName")
self.dataEvRcvrs = None
self.jobReadyEvRcvr = None
self.jobDoneEvRcvr = None
self.jobAcceptedEvRcvr = None
self.jobAssignEvTrx = None
self.jobOfficeRcvr = None
if not forDaemon:
self._setEventPipes()
import sys
import unittest
import time
import copy
from lsst.ctrl.sched.joboffice.jobOffice import JobOffice, _BaseJobOffice, DataTriggeredJobOffice, unserializePolicy, serializePolicy
from lsst.ctrl.sched.blackboard.item import JobItem, DataProductItem
from lsst.ctrl.sched import Dataset
from lsst.pex.policy import Policy, DefaultPolicyFile
from lsst.daf.base import PropertySet
from lsst.ctrl.events import StatusEvent, CommandEvent, EventTransmitter, EventSystem
testdir = os.path.join(os.environ["CTRL_SCHED_DIR"], "tests")
exampledir = os.path.join(os.environ["CTRL_SCHED_DIR"], "examples")
bbdir = os.path.join(testdir, "testbb")
policyFile = DefaultPolicyFile("ctrl_sched", "ccdassembly-joboffice.paf",
"examples")
postisrdata = """#<?cfg paf policy ?>
type: PostISR
ids: {
visitid: 44291
ccdid: 3
raftid: 33
snapid: 0
ampid: 5
}
"""
brokerhost = "lsst8.ncsa.uiuc.edu"
originatorId = EventSystem.getDefaultEventSystem().createOriginatorId()
class AbstractJobOfficeTestCase(unittest.TestCase):
def setUp(self):
example = DefaultPolicyFile("ctrl_sched", "srcAssoc-joboffice.paf",
"examples")
self.schedpolicy = Policy.createPolicy(example)
self.tpolicy = self.schedpolicy.get("schedule.trigger")
self.trigger = None