def testProcessDataset(self):
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 0)
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
# manipulate the policy
idp = Policy.createPolicy(PolicyString(idpolicy))
spolicy.set("job.identity", idp)
self.sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
# pdb.set_trace()
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=15)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 1)
self.assertEquals(self.bb.queues.jobsPossible.length(), 1)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
self.assertEquals(self.sched.nameNumber, 2)
ds = Dataset("PostISR", visitid=95, ccdid=22, snapid=0, ampid=15)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 2)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(1)
self.assertEquals(job.getName(), "Job-2")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
inputs = job.getInputDatasets()
self.assertEquals(len(inputs), 16)
self.assertEquals(inputs[0].type, "PostISR")
self.assertEquals(self.sched.nameNumber, 3)
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=14)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 3)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 14)
# pdb.set_trace()
for i in xrange(14):
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=i)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 17)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
self.assert_(job.isReady())
def main():
"""
run the job office
"""
(cl.opts, cl.args) = cl.parse_args()
if len(cl.args) == 0:
fail("Missing policy file")
if not os.path.exists(cl.args[0]):
fail("%s: policy file not found" % cl.args[0])
if not os.path.exists(cl.opts.rootdir):
fail("%s: root directory not found" % cl.opts.rootdir)
if not cl.opts.runid:
logger.log(Log.WARN, "No RunID given (via -r)")
defpolf = DefaultPolicyFile("ctrl_sched", "JobOffice_dict.paf", "policies")
policy = Policy.createPolicy(cl.args[0])
policy.mergeDefaults(Policy.createPolicy(defpolf,
defpolf.getRepositoryPath()))
name = policy.getString("name")
# set job office root directory
if not os.path.isabs(cl.opts.rootdir):
cl.opts.rootdir = os.path.abspath(cl.opts.rootdir)
persistdir = os.path.join(cl.opts.rootdir, name)
if policy.exists("persist.dir"):
persistdir = policy.get("persist.dir") % \
{"schedroot": cl.opts.rootdir, "name": name }
# create the logger(s)
logfile = cl.opts.logfile
if not logfile:
logfile = os.path.join(persistdir, "joboffice.log")
if not os.path.exists(logfile):
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
if not cl.opts.asdaemon or cl.opts.toscreen:
ofclogger = DualLog(logfile, Log.DEBUG, Log.DEBUG, False)
# logging bug workaround
ofclogger.setScreenVerbose(False)
else:
ofclogger = Log()
ofclogger.addDestination(logfile)
ofclogger.setThreshold(run.verbosity2threshold(cl.opts.logverb, 0))
ofclogger.log(-2,"office threshold: %i" % ofclogger.getThreshold())
try:
# create the JobOffice instance
office = createJobOffice(cl.opts.rootdir, policy, ofclogger,
cl.opts.runid, cl.opts.brokerhost,
cl.opts.brokerport)
except Exception, ex:
logger.log(Log.FATAL, "Failed to create job office: " + str(ex))
raise
sys.exit(1)
def main():
try:
(cl.opts, cl.args) = cl.parse_args();
setVerbosity(cl.opts.verbosity)
nodes = []
if cl.opts.policy is not None:
policy = Policy.createPolicy(cl.opts.policy)
nodes = getHeadNodes(policy)
nodes.extend(cl.args)
logger.log(Log.DEBUG, "Killing pipelines on " + ", ".join(nodes))
remcmd = "%s %s" % \
(os.path.join(os.environ[pkgdirvar], "bin", remkill),cl.opts.runid)
remcmd = remcmd.strip()
for node in nodes:
cmd = ("ssh", node, remcmd)
logger.log(Log.INFO, "executing: %s %s '%s'" % cmd)
if subprocess.call(cmd) != 0:
logger.log(Log.WARN, "Failed to kill processes on " + node)
except:
tb = traceback.format_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
logger.log(Log.FATAL, tb[-1].strip())
logger.log(Log.DEBUG, "".join(tb[0:-1]).strip())
sys.exit(1)
sys.exit(0)
def setup(self, policyDict="DataReady_dict.paf"):
deffile = DefaultPolicyFile("ctrl_sched", policyDict, "policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self,"policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("inputKeys.completedDatasets")
self.clipboardKeys["possibleDatasets"] = \
self.policy.getString("inputKeys.possibleDatasets")
self.dataclients = []
clpols = []
if self.policy.exists("datasets"):
clpols = self.policy.getPolicyArray("datasets")
for pol in clpols:
dstype = None
if pol.exists("datasetType"):
dstype = pol.getString("datasetType")
topic = pol.getString("dataReadyEvent")
reportAll = pol.getBool("reportAllPossible")
client = DataReadyClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), dstype,
reportAll)
self.dataclients.append(client)
def testSetAdd(self):
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_good.paf"),
"", True)
self.assert_(p.canValidate())
self.assertValidationError(
ValidationError.TOO_MANY_VALUES,
p.add, "bool_set_count", True)
self.assert_(p.valueCount("bool_set_count") == 1)
self.assertValidationError(
ValidationError.VALUE_DISALLOWED,
p.set, "bool_set_count", False)
self.assert_(p.getBool("bool_set_count") == True)
p.set("int_range_count", -7)
self.assertValidationError(
ValidationError.VALUE_OUT_OF_RANGE,
p.add, "int_range_count", 10)
# add & set don't check against minOccurs, but validate() does
try:
p.validate()
except ValidationError as ve:
self.assert_(ve.getErrors("int_range_count")
== ValidationError.NOT_AN_ARRAY)
self.assert_(ve.getErrors("required")
== ValidationError.MISSING_REQUIRED)
p.add("int_range_count", -8)
p.set("required", "foo")
p.validate()
def testPolicyCopy(self):
p = Policy.createPolicy("examples/EventTransmitter_policy.paf")
pp = Policy(p, True)
self.assertEquals(p.get("transmitter.serializationFormat"), "deluxe")
self.assertEquals(pp.getString("transmitter.serializationFormat"), "deluxe")
p = None
self.assertEquals(pp.getString("transmitter.serializationFormat"), "deluxe")
def testSelfValidation(self):
# assign a dictionary after creation
p = Policy(self.getTestDictionary("types_policy_good.paf"))
p.loadPolicyFiles(self.getTestDictionary(), True)
typesDict = Dictionary(self.getTestDictionary("types_dictionary.paf"))
valuesDict = Dictionary(self.getTestDictionary("values_dictionary.paf"))
self.assert_(not p.canValidate())
p.setDictionary(typesDict)
self.assert_(p.canValidate())
p.validate()
p.set("bool_type", True)
self.assertValidationError(
ValidationError.WRONG_TYPE, p.set, "bool_type", "foo")
# create with dictionary
p = Policy.createPolicy(self.getTestDictionary("types_dictionary.paf"), "", True)
self.assert_(p.canValidate())
p.set("bool_type", True)
self.assertValidationError(
ValidationError.WRONG_TYPE, p.set, "bool_type", "foo")
# assign a dictionary after invalid modifications
p = Policy()
p.set("bool_type", "foo")
p.setDictionary(typesDict)
ve = ValidationError("Dictionary_1.py", 1, "testSelfValidation")
p.validate(ve)
self.assert_(ve.getErrors("bool_type") == ValidationError.WRONG_TYPE)
try:
p.validate()
except ValidationError, e:
ve = e.args[0]
self.assert_(ve.getErrors("bool_type") == ValidationError.WRONG_TYPE)
self.assert_(ve.getParamCount() == 1)
def extractIncludedFilenames(prodPolicyFile, repository=".",
pipefile=None, logger=None):
"""
extract all the filenames included, directly or indirectly, from the
given policy file. When a repository is provided, included files will
be recursively opened and searched. The paths in the returned set
will not include the repository directory. Use pipefile to skip
the inclusion of pipeline policy files.
@param prodPolicyFile the policy file to examine. This must
be the full path to the file
@param repository the policy repository. If None, the current
directory is assumed.
@param pipefile the hierarchical policy name for a pipeline
definition. Any included policy filenames
at this node or lower will not be added.
@param logger if provided, use this Log to record any
warnings about missing or bad files;
otherwise, problems are silently ignored.
@return set containing the unique set of policy filenames found,
including the given top file and the
"""
prodPolicy = Policy.createPolicy(prodPolicyFile, False)
filenames = set([prodPolicyFile])
ProvenanceSetup._listFilenames(filenames, prodPolicy, None, repository,
pipefile, logger)
filenames.discard(prodPolicyFile)
return filenames
def setup(self):
deffile = DefaultPolicyFile("ctrl_sched", "GetAJob_dict.paf",
"policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self, "policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
self.jobid = None
self.tagLogger(None)
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["jobIdentity"] = \
self.policy.getString("outputKeys.jobIdentity")
self.clipboardKeys["inputDatasets"] = \
self.policy.getString("outputKeys.inputDatasets")
self.clipboardKeys["outputDatasets"] = \
self.policy.getString("outputKeys.outputDatasets")
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("outputKeys.completedDatasets")
self.log.log(Log.INFO - 1,
"clipboard keys: " + str(self.clipboardKeys))
topic = self.policy.getString("pipelineEvent")
self.client = GetAJobClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), self.log)
self.log.log(Log.INFO - 1,
"Using OriginatorId = %d" % self.client.getOriginatorId())
def setup(self, policyDict="DataReady_dict.paf"):
deffile = DefaultPolicyFile("ctrl_sched", policyDict, "policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self, "policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("inputKeys.completedDatasets")
self.clipboardKeys["possibleDatasets"] = \
self.policy.getString("inputKeys.possibleDatasets")
self.dataclients = []
clpols = []
if self.policy.exists("datasets"):
clpols = self.policy.getPolicyArray("datasets")
for pol in clpols:
dstype = None
if pol.exists("datasetType"):
dstype = pol.getString("datasetType")
topic = pol.getString("dataReadyEvent")
reportAll = pol.getBool("reportAllPossible")
client = DataReadyClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), dstype,
reportAll)
self.dataclients.append(client)
def testPolicyCopy(self):
p = Policy.createPolicy(os.path.join(proddir, "examples", "EventTransmitter_policy.paf"))
pp = Policy(p, True)
self.assertEqual(p.get("transmitter.serializationFormat"), "deluxe")
self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
p = None
self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
def testPolicyCopy(self):
p = Policy.createPolicy(os.path.join(proddir, "examples", "EventTransmitter_policy.paf"))
pp = Policy(p, True)
self.assertEqual(p.get("transmitter.serializationFormat"), "deluxe")
self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
p = None
self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
def setup(self):
deffile = DefaultPolicyFile("ctrl_sched","GetAJob_dict.paf","policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self,"policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
self.jobid = None
self.tagLogger(None)
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["jobIdentity"] = \
self.policy.getString("outputKeys.jobIdentity")
self.clipboardKeys["inputDatasets"] = \
self.policy.getString("outputKeys.inputDatasets")
self.clipboardKeys["outputDatasets"] = \
self.policy.getString("outputKeys.outputDatasets")
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("outputKeys.completedDatasets")
self.log.log(Log.INFO-1, "clipboard keys: " + str(self.clipboardKeys))
topic = self.policy.getString("pipelineEvent")
self.client = GetAJobClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), self.log)
self.log.log(Log.INFO-1,
"Using OriginatorId = %d" % self.client.getOriginatorId())
def testCreateName(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
ds = Dataset("PostISR", ampid=3)
self.assertEquals(sched.createName(ds), "Job-1")
def testCreateName2(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
spolicy.set("job.name.template", "%(type)s-v%(ampid)s")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
ds = Dataset("PostISR", ampid=3)
self.assertEquals(sched.createName(ds), "PostISR-v3")
def testSelfValidation(self):
# assign a dictionary after creation
p = Policy(self.getTestDictionary("types_policy_good.paf"))
p.loadPolicyFiles(self.getTestDictionary(), True)
typesDict = Dictionary(self.getTestDictionary("types_dictionary.paf"))
valuesDict = Dictionary(
self.getTestDictionary("values_dictionary.paf"))
self.assertFalse(p.canValidate())
p.setDictionary(typesDict)
self.assertTrue(p.canValidate())
p.validate()
p.set("bool_type", True)
self.assertValidationError(ValidationError.WRONG_TYPE, p.set,
"bool_type", "foo")
# create with dictionary
p = Policy.createPolicy(self.getTestDictionary("types_dictionary.paf"),
"", True)
self.assertTrue(p.canValidate())
p.set("bool_type", True)
self.assertValidationError(ValidationError.WRONG_TYPE, p.set,
"bool_type", "foo")
# assign a dictionary after invalid modifications
p = Policy()
p.set("bool_type", "foo")
p.setDictionary(typesDict)
ve = ValidationError("Dictionary_1.py", 1, "testSelfValidation")
p.validate(ve.cpp)
self.assertEqual(ve.getErrors("bool_type"), ValidationError.WRONG_TYPE)
try:
p.validate()
except ValidationError as ve:
self.assertEqual(ve.getErrors("bool_type"),
ValidationError.WRONG_TYPE)
self.assertEqual(ve.getParamCount(), 1)
p.set("bool_type", True)
p.set("int_type", 1)
p.validate()
# switch dictionaries
oldD = p.getDictionary()
p.setDictionary(valuesDict)
try:
p.validate()
except ValidationError as ve:
self.assertEqual(ve.getErrors("bool_type"),
ValidationError.UNKNOWN_NAME)
p.set("string_range_type", "moo")
try:
p.set("string_range_type", "victor")
except ValidationError as ve:
self.assertEqual(ve.getErrors("string_range_type"),
ValidationError.VALUE_OUT_OF_RANGE)
p.setDictionary(oldD)
p.remove("string_range_type")
p.validate()
def setUp(self):
policy = Policy.createPolicy(policyFile)
# pdb.set_trace()
self.joboffice = DataTriggeredJobOffice(testdir,
policy=policy,
log=None,
runId="testing",
brokerHost=brokerhost)
self.joboffice.log.setThreshold(self.joboffice.log.WARN)
self.jodir = os.path.join(testdir, "ccdassembly")
def __init__(self, policyfile=None):
"""
create an item with the given properties.
@param policyfile A policy
"""
# the properties attached to this items
self._props = None
if policyfile:
self._props = Policy.createPolicy(policyfile)
else:
self._props = Policy()
def testSelfValidation(self):
# assign a dictionary after creation
p = Policy(self.getTestDictionary("types_policy_good.paf"))
p.loadPolicyFiles(self.getTestDictionary(), True)
typesDict = Dictionary(self.getTestDictionary("types_dictionary.paf"))
valuesDict = Dictionary(self.getTestDictionary("values_dictionary.paf"))
self.assert_(not p.canValidate())
p.setDictionary(typesDict)
self.assert_(p.canValidate())
p.validate()
p.set("bool_type", True)
self.assertValidationError(
ValidationError.WRONG_TYPE, p.set, "bool_type", "foo")
# create with dictionary
p = Policy.createPolicy(self.getTestDictionary("types_dictionary.paf"), "", True)
self.assert_(p.canValidate())
p.set("bool_type", True)
self.assertValidationError(
ValidationError.WRONG_TYPE, p.set, "bool_type", "foo")
# assign a dictionary after invalid modifications
p = Policy()
p.set("bool_type", "foo")
p.setDictionary(typesDict)
ve = ValidationError("Dictionary_1.py", 1, "testSelfValidation")
p.validate(ve.cpp)
self.assert_(ve.getErrors("bool_type") == ValidationError.WRONG_TYPE)
try:
p.validate()
except ValidationError as ve:
self.assert_(ve.getErrors("bool_type") == ValidationError.WRONG_TYPE)
self.assert_(ve.getParamCount() == 1)
p.set("bool_type", True)
p.set("int_type", 1)
p.validate()
# switch dictionaries
oldD = p.getDictionary()
p.setDictionary(valuesDict)
try:
p.validate()
except ValidationError as ve:
self.assert_(ve.getErrors("bool_type")
== ValidationError.UNKNOWN_NAME)
p.set("string_range_type", "moo")
try:
p.set("string_range_type", "victor")
except ValidationError as ve:
self.assert_(ve.getErrors("string_range_type")
== ValidationError.VALUE_OUT_OF_RANGE)
p.setDictionary(oldD)
p.remove("string_range_type")
p.validate()
def __init__(self, policyfile=None):
"""
create an item with the given properties.
@param policyfile A policy
"""
# the properties attached to this items
self._props = None
if policyfile:
self._props = Policy.createPolicy(policyfile)
else:
self._props = Policy()
def testCtor(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
self.assert_(sched.nametmpl is None)
self.assertEquals(sched.defaultName, "Job")
self.assertEquals(sched.nameNumber, 1)
self.assertEquals(len(sched.triggers), 1)
self.assertEquals(len(sched.inputdata), 1)
def testDefaults(self):
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_good.paf"),
"", True)
self.assert_(p.valueCount("bool_set_count") == 1)
self.assert_(p.getBool("bool_set_count") == True)
self.assert_(p.valueCount("int_range_count") == 3)
self.assert_(p.getDouble("deep.sub_double") == 1.)
self.assert_(p.get("indirect4.string_type") == "foo")
try:
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_bad_1.paf"),
"", True)
except ValidationError as ve:
self.assert_(ve.getErrors("double")
== ValidationError.WRONG_TYPE)
self.assert_(ve.getErrors("int_range_count")
== ValidationError.NOT_AN_ARRAY)
self.assert_(ve.getErrors("bool_set_count")
== ValidationError.TOO_MANY_VALUES)
self.assert_(ve.getErrors("deep.sub_double")
== ValidationError.WRONG_TYPE)
self.assert_(ve.getParamCount() == 4)
def testDefaults(self):
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_good.paf"),
"", True)
self.assertEqual(p.valueCount("bool_set_count"), 1)
self.assertIs(p.getBool("bool_set_count"), True)
self.assertEqual(p.valueCount("int_range_count"), 3)
self.assertEqual(p.getDouble("deep.sub_double"), 1.)
self.assertEqual(p.get("indirect4.string_type"), "foo")
try:
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_bad_1.paf"),
"", True)
except ValidationError as ve:
self.assertEqual(ve.getErrors("double"),
ValidationError.WRONG_TYPE)
self.assertEqual(ve.getErrors("int_range_count"),
ValidationError.NOT_AN_ARRAY)
self.assertEqual(ve.getErrors("bool_set_count"),
ValidationError.TOO_MANY_VALUES)
self.assertEqual(ve.getErrors("deep.sub_double"),
ValidationError.WRONG_TYPE)
self.assertEqual(ve.getParamCount(), 4)
def _shallowPolicyNodeResolve(pname, policy, repository, logger=None):
if not policy.exists(pname):
return []
nodes = policy.getArray(pname)
if policy.isFile(pname):
for i in range(len(nodes)):
try:
if not os.path.isabs(nodes[i]):
nodes[i] = os.path.join(repository, nodes[i])
nodes[i] = Policy.createPolicy(nodes[i], False)
except lsst.pex.exceptions.Exception:
if logger:
logger.log(Log.WARN, "problem finding/loading " + nodes[i])
nodes[i] = None
return nodes
def __init__(self, blackboard, policy, logger=None):
"""
create the scheduler
@param blackboard the blackboard that this scheduler will update.
@param policy the "schedule" policy used to configure this
scheduler
@param logger a log to use for messages
"""
Scheduler.__init__(self, blackboard, logger, True)
defpol = DefaultPolicyFile("ctrl_sched",
"DataTriggeredScheduler_dict.paf",
"policies")
defpol = Policy.createPolicy(defpol, defpol.getRepositoryPath(), False)
policy.mergeDefaults(defpol)
self.triggers = []
trigps = policy.getArray("trigger")
for trigp in trigps:
self.triggers.append(Trigger.fromPolicy(trigp))
self.inputdata = []
inpps = policy.getArray("job.input")
for dsp in inpps:
self.inputdata.append(Trigger.fromPolicy(dsp, True))
self.outputdata = []
outpps = policy.getArray("job.output")
for dsp in outpps:
self.outputdata.append(Trigger.fromPolicy(dsp, True))
self.jobIdConf = None
if policy.exists("job.identity"):
self.jobIdConf = policy.getPolicy("job.identity")
self.nametmpl = None
pol = policy.get("job.name")
self.defaultName = pol.getString("default")
if pol.exists("template"):
self.nametmpl = pol.getString("template")
self.nameNumber = pol.getInt("initCounter")
self.jobRetries = None
if policy.exists("job.retries"):
self.jobRetries = policy.getInt("job.retries")
def testSampleCode(self):
policyFile = DefaultPolicyFile("pex_policy", "defaults_dictionary_complete.paf",
"tests/dictionary")
defaults = Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True)
policy = Policy()
policy.mergeDefaults(defaults)
self.assert_(policy.canValidate())
policy = Policy()
policy.mergeDefaults(defaults, False)
self.assert_(not policy.canValidate())
# even if not keeping it around for future validation, validate the merge now
policy = Policy()
policy.set("bad", "worse")
self.assertValidationError(ValidationError.UNKNOWN_NAME,
policy.mergeDefaults, defaults, False)
self.assert_(not policy.canValidate())
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not StringIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "StringIDFilter_dict.paf",
"policies")
StringIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if StringIDFilter._dictionary:
p.mergeDefaults(StringIDFilter._dictionary)
name = "unknown"
vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("value"): vals = policy.getArray("value")
return StringIDFilter(name, vals)
def testSampleCode(self):
policyFile = DefaultPolicyFile("pex_policy", "defaults_dictionary_complete.paf",
"tests/dictionary")
defaults = Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True)
policy = Policy()
policy.mergeDefaults(defaults)
self.assertTrue(policy.canValidate())
policy = Policy()
policy.mergeDefaults(defaults, False)
self.assertFalse(policy.canValidate())
# even if not keeping it around for future validation, validate the merge now
policy = Policy()
policy.set("bad", "worse")
self.assertValidationError(ValidationError.UNKNOWN_NAME,
policy.mergeDefaults, defaults, False)
self.assertFalse(policy.canValidate())
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not StringIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "StringIDFilter_dict.paf",
"policies")
StringIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if StringIDFilter._dictionary:
p.mergeDefaults(StringIDFilter._dictionary)
name = "unknown"
vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("value"): vals = policy.getArray("value")
return StringIDFilter(name, vals)
def launchDC3a(policyFile, runid, visitFiles, colls, opts, logger):
if not os.environ.has_key(pkgdirvar):
raise pexExcept.LsstException("%s env. var not set (setup %s)"
% (pkgdirvar, dc3apkg))
if opts.repos is None:
opts.repos = os.path.join(os.environ[pkgdirvar], "pipeline")
policy = Policy.createPolicy(policyFile, opts.repos)
broker = policy.get("eventBrokerHost")
logger.log(Log.DEBUG, "Using event broker on %s" % broker)
recvr = events.EventReceiver(broker, loggingEventTopic)
runOrca(policyFile, runid, opts, logger)
waitForReady(policy, runid, recvr, opts.pipeverb, logger)
runEventGen(policy, visitFiles, colls, opts, broker, logger)
def launchMos(policyFile, runid, visitFiles, colls, opts, logger):
if not os.environ.has_key(pkgdirvar):
raise pexExcept.LsstException("%s env. var not set (setup %s)"
% (pkgdirvar, mospkg))
if opts.repos is None:
opts.repos = os.path.join(os.environ[pkgdirvar], "pipeline")
policy = Policy.createPolicy(policyFile, opts.repos)
broker = policy.get("eventBrokerHost")
logger.log(Log.DEBUG, "Using event broker on %s" % broker)
print >> sys.stderr, "Using event broker on %s" % broker
recvr = events.EventReceiver(broker, loggingEventTopic)
runOrca(policyFile, runid, opts, logger)
waitForReady(policy, runid, recvr, opts.pipeverb, logger)
runEventGen(policy, visitFiles, colls, opts, broker, logger)
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not IntegerIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "IntegerIDFilter_dict.paf",
"policies")
IntegerIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if IntegerIDFilter._dictionary:
p.mergeDefaults(IntegerIDFilter._dictionary)
name = "unknown"
min = lim = vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("min"): min = policy.getInt("min")
if policy.exists("lim"): lim = policy.getInt("lim")
if policy.exists("value"): vals = policy.getArray("value")
return IntegerIDFilter(name, min, lim, vals)
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not IntegerIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "IntegerIDFilter_dict.paf",
"policies")
IntegerIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if IntegerIDFilter._dictionary:
p.mergeDefaults(IntegerIDFilter._dictionary)
name = "unknown"
min = lim = vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("min"): min = policy.getInt("min")
if policy.exists("lim"): lim = policy.getInt("lim")
if policy.exists("value"): vals = policy.getArray("value")
return IntegerIDFilter(name, min, lim, vals)
def extractPipelineFilenames(wfname, prodPolicyFile, repository=".",
logger=None):
"""
extract all non-pipeline policy files in the given production
policy file.
@param wfname the name of the workflow of interest
@param prodPolicyFile the production-level policy file
@param repository the policy repository
@param logger if provided, use this Log to record any
warnings about missing or bad files;
otherwise, problems are silently ignored.
"""
prodPolicy = Policy.createPolicy(prodPolicyFile, False)
out = []
wfs = ProvenanceSetup._shallowPolicyNodeResolve("workflow", prodPolicy,
repository, logger)
if not wfs:
return out
for wfp in wfs:
if wfp is None:
continue
if not wfp.exists("shortName") or wfp.get("shortName") != wfname:
continue
pipes = ProvenanceSetup._shallowPolicyNodeResolve("pipeline",
wfp, repository)
for pipe in pipes:
if not pipe.exists("definition") or not pipe.isFile("definition"):
continue
pipe = pipe.get("definition").getPath()
out.append(pipe)
pipe = os.path.join(repository, pipe)
if not os.path.exists(pipe):
if logger:
logger.log(Log.WARN, "Policy file not found in repository: "+pipe)
continue
out += list(ProvenanceSetup.extractIncludedFilenames(pipe,
repository))
return out
def recordPolicy(self, policyFile):
"""Record the contents of the given Policy as part of provenance."""
md5 = hashlib.md5()
f = open(policyFile, 'r')
for line in f:
md5.update(line)
f.close()
# self._realRecordPolicyFile(self._rundb, policyFile, md5)
self._realRecordPolicyFile(self._globalDb, policyFile, md5)
p = Policy.createPolicy(policyFile, False)
for key in p.paramNames():
type = p.getTypeName(key)
val = p.str(key) # works for arrays, too
val = re.sub(r'\0', r'', val) # extra nulls get included
# self._realRecordPolicyContents(self._rundb, key, type, val)
self._realRecordPolicyContents(self._globalDb, key, type, val)
self._policyKeyId += 1
self._policyFileId += 1
def recordPolicy(self, policyFile):
"""Record the contents of the given Policy as part of provenance."""
md5 = hashlib.md5()
f = open(policyFile, 'r')
for line in f:
md5.update(line)
f.close()
# self._realRecordPolicyFile(self._rundb, policyFile, md5)
self._realRecordPolicyFile(self._globalDb, policyFile, md5)
p = Policy.createPolicy(policyFile, False)
for key in p.paramNames():
type = p.getTypeName(key)
val = p.str(key) # works for arrays, too
val = re.sub(r'\0', r'', val) # extra nulls get included
# self._realRecordPolicyContents(self._rundb, key, type, val)
self._realRecordPolicyContents(self._globalDb, key, type, val)
self._policyKeyId += 1
self._policyFileId += 1
def _listFilenames(fileset, policy, basename, repository, stopname=None,
logger=None):
if stopname and basename and not stopname.startswith(basename):
stopname = None
for name in policy.names(True):
fullname = basename and ".".join([basename, name]) or name
if stopname and fullname == stopname:
continue
if policy.isFile(name):
files = policy.getArray(name)
for file in files:
file = file.getPath()
if file not in fileset:
fileset.add(file)
file = os.path.join(repository, file)
if not os.path.exists(file):
if logger:
logger.log(logger.WARN, "Policy file not found in repository: %s" % file)
continue
try:
if logger and logger.sends(Log.DEBUG):
logger.log(Log.DEBUG, "opening log file: %s" % file)
fpolicy = Policy.createPolicy(file, False)
ProvenanceSetup._listFilenames(fileset, fpolicy,
fullname, repository,
stopname, logger)
except lsst.pex.exceptions.Exception as ex:
if logger:
logger.log(Log.WARN, "problem loading %s: %s" % (file, str(ex)))
continue
elif policy.isPolicy(name):
pols = policy.getArray(name)
for pol in pols:
ProvenanceSetup._listFilenames(fileset, pol, fullname,
repository, stopname, logger)
def testSetAdd(self):
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_good.paf"),
"", True)
self.assertTrue(p.canValidate())
self.assertValidationError(ValidationError.TOO_MANY_VALUES,
p.add, "bool_set_count", True)
self.assertEqual(p.valueCount("bool_set_count"), 1)
self.assertValidationError(ValidationError.VALUE_DISALLOWED,
p.set, "bool_set_count", False)
self.assertIs(p.getBool("bool_set_count"), True)
p.set("int_range_count", -7)
self.assertValidationError(ValidationError.VALUE_OUT_OF_RANGE,
p.add, "int_range_count", 10)
# add & set don't check against minOccurs, but validate() does
try:
p.validate()
except ValidationError as ve:
self.assertEqual(ve.getErrors("int_range_count"),
ValidationError.NOT_AN_ARRAY)
self.assertEqual(ve.getErrors("required"),
ValidationError.MISSING_REQUIRED)
p.add("int_range_count", -8)
p.set("required", "foo")
p.validate()
def launchDC2(policyFile, runid, exposureFiles):
if not os.environ.has_key(pkgdirvar):
raise RuntimeError("DC2PIPE_DIR env. var not set (setup dc2pipe)")
# load the launching policy file
if not os.path.exists(policyFile):
raise RuntimeError(policyFile + ": policy file not found")
pol = Policy.createPolicy(policyFile)
if cl.opts.policyRepository is not None:
pol.set("policyRepository", cl.opts.policyRepository)
# find the policy repository
defRepository = os.path.join(os.environ[pkgdirvar], "pipeline")
repository = pol.get("policyRepository", defRepository)
if not os.path.exists(repository):
raise RuntimeError(repository + ": directory not found")
if not os.path.isdir(repository):
raise RuntimeError(repository + ": not a directory")
# set the exposure lists
if len(exposureFiles) == 0:
exposureFiles = pol.getArray("exposureList", [])
if len(exposureFiles) == 0:
raise RuntimeError("No exposure list files specified in policy")
# determine the parent of the working directories
home = pol.get("workingHome", "/share/DC2root")
if not os.path.exists(home):
raise RuntimeError(home + ": working home directory not found")
if not cl.opts.forceRunId and os.path.exists(os.path.join(home, runid)):
raise RuntimeError("Run ID already used (use -f to override)")
# deploy each pipeline: create nodelist.scr, copy over policy files, etc.
pipepol = pol.get("pipelines")
pipelines = pipepol.policyNames(True)
workingdir = {}
masternode = {}
envscript = {}
willrun = []
for pipeline in pipelines:
ppol = pipepol.get(pipeline)
if ppol.get("launch", 1) != 0:
(masternode[pipeline], workingdir[pipeline], envscript[pipeline]) \
= prepPipeline(pipeline, ppol, runid, home, repository)
willrun.append(pipeline)
# create the database tables for this run
if cl.opts.createDB: createDatabase(runid)
# now launch each pipeline
for pipeline in willrun:
ppol = pipepol.get(pipeline)
launchPipeline(pipeline, runid, masternode[pipeline],
workingdir[pipeline], envscript[pipeline])
# send input data events
for efile in exposureFiles:
if not os.path.isabs(efile):
if os.path.exists(efile) and not efile.startswith("."):
efile = os.path.join(".", efile)
elif os.path.exists(
os.path.join(os.environ[pkgdirvar], "exposureLists",
efile)):
efile = os.path.join(os.environ[pkgdirvar], "exposureLists",
efile)
if not os.path.exists(efile):
logger.log(Log.WARN, "Exposure list file not found: " + efile)
logger.log(Log.DEBUG, "Pausing for 15s, waiting for setup...")
time.sleep(15)
logger.log(Log.INFO, "Sending exposure data from " + efile)
logger.log(Log.DEBUG, "executing: %s < %s" % (eventgenerator, efile))
with file(efile) as expfile:
if subprocess.call(eventgenerator.split(), stdin=expfile) != 0:
raise RuntimeError("Failed to execute eventgenerator")
def setUp(self):
self.policyfile = os.path.join(proddir, "examples", "types.paf")
self.policy = Policy.createPolicy(self.policyfile, False)
def setUp(self):
self.policyfile = os.path.join(proddir,"examples","types.paf")
self.policy = Policy.createPolicy(self.policyfile, False)
def __init__(self,
rootdir,
policy=None,
defPolicyFile=None,
log=None,
runId=None,
brokerHost=None,
brokerPort=None,
forDaemon=False,
fromSubclass=False):
"""
create the JobOffice
@param rootdir the root directory where job offices may set up
its blackboard data. This JobOffice will
create a subdirectory here for its data with
a name set by the "name" policy paramter.
@param policy the policy to use to configure this JobOffice
@param defPolicyFile the DefaultPolicyFile to use for defaults. If
this points to a dictionary, a policy
validation will be done. If None, an
internally identified default policy file
will be used.
@param log a logger to use; if None, the default logger will
will be used. A child log will be created.
@param runId the run ID to restrict our attention to. If
not None, incoming events will be restricted
to the given runId.
@param brokerHost the machine where the event broker is running.
If None (default), the host given in the
policy is used. This parameter is for
carrying an override from the command line.
@param brokerPort the port to use to connect to the event broker.
If None (default), the port given in the
policy is used. This parameter is for
carrying an override from the command line.
@param forDaemon if true, the creation of the Event channels will
be delayed until run() is called. This allows
the caller to fork as needed without losing
the connections with the event broker. This
also means that some public functions (other
than run() and start()) will not work
properly until ensureReady() is called.
@param fromSubclass the flag indicating that this constructor is
being properly called. Calls to this
constructor from a subclass constructor should
set this to True.
"""
self._checkAbstract(fromSubclass, "_BasicJobOffice")
# start by establishing policy data
if not defPolicyFile:
defPolicyFile = DefaultPolicyFile("ctrl_sched",
"baseJobOffice_dict.paf",
"policies")
defaults = Policy.createPolicy(defPolicyFile,
defPolicyFile.getRepositoryPath(), True)
if not policy:
policy = Policy()
self.policy = policy
if defaults.canValidate():
self.policy.mergeDefaults(defaults.getDictionary())
else:
self.policy.mergeDefaults(defaults)
# instantiate parent class
name = self.policy.get("name")
persistDir = self.policy.get("persist.dir") % {
"schedroot": rootdir,
"name": name
}
if not os.path.exists(persistDir):
os.makedirs(persistDir)
JobOffice.__init__(self, persistDir, log, runId, True)
self.setName(name)
# logger
self.log = Log(self.log, self.name)
# initialize some data from policy
self.initialWait = self.policy.get("listen.initialWait")
self.emptyWait = self.policy.get("listen.emptyWait")
self.dataTopics = self.policy.getArray("listen.dataReadyEvent")
self.jobTopic = self.policy.get("listen.pipelineEvent")
self.stopTopic = self.policy.get("listen.stopEvent")
self.jobOfficeTopic = self.policy.get("listen.jobOfficeEvent")
self.jobOfficeStatusTopic = self.policy.get("listen.jobOfficeStatus")
self.highWatermark = self.policy.get("listen.highWatermark")
# initialize the event system
self.jobReadyEvRcvr = self.dataEvRcvrs = None
self.jobDoneEvRcvr = self.jobAcceptedEvRcvr = None
self.brokerHost = brokerHost
self.brokerPort = brokerPort
if not self.brokerPort and self.policy.exists("listen.brokerHostPort"):
self.brokerPort = self.policy.get("listen.brokerHostPort")
if not self.brokerHost and (not self.brokerPort
or self.brokerPort > 0):
self.brokerHost = self.policy.get("listen.brokerHostName")
self.dataEvRcvrs = None
self.jobReadyEvRcvr = None
self.jobDoneEvRcvr = None
self.jobAcceptedEvRcvr = None
self.jobAssignEvTrx = None
self.jobOfficeRcvr = None
if not forDaemon:
self._setEventPipes()
def testRead(self):
ps = PolicyString(self.data)
p = Policy.createPolicy(ps)
self.assertEqual(p.get("int"), 7)
self.assertEqual(p.get("dbl"), -1.0)
def openItem(self, filename):
out = PolicyBlackboardItem()
out._props = Policy.createPolicy(filename)
return out
def __init__(self, rootdir, policy=None, defPolicyFile=None, log=None,
runId=None, brokerHost=None, brokerPort=None, forDaemon=False,
fromSubclass=False):
"""
create the JobOffice
@param rootdir the root directory where job offices may set up
its blackboard data. This JobOffice will
create a subdirectory here for its data with
a name set by the "name" policy paramter.
@param policy the policy to use to configure this JobOffice
@param defPolicyFile the DefaultPolicyFile to use for defaults. If
this points to a dictionary, a policy
validation will be done. If None, an
internally identified default policy file
will be used.
@param log a logger to use; if None, the default logger will
will be used. A child log will be created.
@param runId the run ID to restrict our attention to. If
not None, incoming events will be restricted
to the given runId.
@param brokerHost the machine where the event broker is running.
If None (default), the host given in the
policy is used. This parameter is for
carrying an override from the command line.
@param brokerPort the port to use to connect to the event broker.
If None (default), the port given in the
policy is used. This parameter is for
carrying an override from the command line.
@param forDaemon if true, the creation of the Event channels will
be delayed until run() is called. This allows
the caller to fork as needed without losing
the connections with the event broker. This
also means that some public functions (other
than run() and start()) will not work
properly until ensureReady() is called.
@param fromSubclass the flag indicating that this constructor is
being properly called. Calls to this
constructor from a subclass constructor should
set this to True.
"""
self._checkAbstract(fromSubclass, "_BasicJobOffice")
# start by establishing policy data
if not defPolicyFile:
defPolicyFile = DefaultPolicyFile("ctrl_sched",
"baseJobOffice_dict.paf",
"policies")
defaults = Policy.createPolicy(defPolicyFile,
defPolicyFile.getRepositoryPath(),
True)
if not policy:
policy = Policy()
self.policy = policy
if defaults.canValidate():
self.policy.mergeDefaults(defaults.getDictionary())
else:
self.policy.mergeDefaults(defaults)
# instantiate parent class
name = self.policy.get("name")
persistDir = self.policy.get("persist.dir") % {"schedroot": rootdir,
"name": name }
if not os.path.exists(persistDir):
os.makedirs(persistDir)
JobOffice.__init__(self, persistDir, log, runId, True)
self.setName(name)
# logger
self.log = Log(self.log, self.name)
# initialize some data from policy
self.initialWait = self.policy.get("listen.initialWait")
self.emptyWait = self.policy.get("listen.emptyWait")
self.dataTopics = self.policy.getArray("listen.dataReadyEvent")
self.jobTopic = self.policy.get("listen.pipelineEvent")
self.stopTopic = self.policy.get("listen.stopEvent")
self.jobOfficeTopic = self.policy.get("listen.jobOfficeEvent")
self.jobOfficeStatusTopic = self.policy.get("listen.jobOfficeStatus")
self.highWatermark = self.policy.get("listen.highWatermark")
# initialize the event system
self.jobReadyEvRcvr = self.dataEvRcvrs = None
self.jobDoneEvRcvr = self.jobAcceptedEvRcvr = None
self.brokerHost = brokerHost
self.brokerPort = brokerPort
if not self.brokerPort and self.policy.exists("listen.brokerHostPort"):
self.brokerPort = self.policy.get("listen.brokerHostPort")
if not self.brokerHost and (not self.brokerPort or self.brokerPort > 0):
self.brokerHost = self.policy.get("listen.brokerHostName")
self.dataEvRcvrs = None
self.jobReadyEvRcvr = None
self.jobDoneEvRcvr = None
self.jobAcceptedEvRcvr = None
self.jobAssignEvTrx = None
self.jobOfficeRcvr = None
if not forDaemon:
self._setEventPipes()
def launchDC2(policyFile, runid, exposureFiles):
if not os.environ.has_key(pkgdirvar):
raise RuntimeError("DC2PIPE_DIR env. var not set (setup dc2pipe)")
# load the launching policy file
if not os.path.exists(policyFile):
raise RuntimeError(policyFile + ": policy file not found")
pol = Policy.createPolicy(policyFile)
if cl.opts.policyRepository is not None:
pol.set("policyRepository", cl.opts.policyRepository);
# find the policy repository
defRepository = os.path.join(os.environ[pkgdirvar], "pipeline")
repository = pol.get("policyRepository", defRepository)
if not os.path.exists(repository):
raise RuntimeError(repository + ": directory not found");
if not os.path.isdir(repository):
raise RuntimeError(repository + ": not a directory");
# set the exposure lists
if len(exposureFiles) == 0:
exposureFiles = pol.getArray("exposureList", [])
if len(exposureFiles) == 0:
raise RuntimeError("No exposure list files specified in policy")
# determine the parent of the working directories
home = pol.get("workingHome", "/share/DC2root")
if not os.path.exists(home):
raise RuntimeError(home + ": working home directory not found")
if not cl.opts.forceRunId and os.path.exists(os.path.join(home, runid)):
raise RuntimeError("Run ID already used (use -f to override)")
# deploy each pipeline: create nodelist.scr, copy over policy files, etc.
pipepol = pol.get("pipelines")
pipelines = pipepol.policyNames(True)
workingdir = {}
masternode = {}
envscript = {}
willrun = []
for pipeline in pipelines:
ppol = pipepol.get(pipeline)
if ppol.get("launch", 1) != 0:
(masternode[pipeline], workingdir[pipeline], envscript[pipeline]) \
= prepPipeline(pipeline, ppol, runid, home, repository)
willrun.append(pipeline)
# create the database tables for this run
if cl.opts.createDB: createDatabase(runid)
# now launch each pipeline
for pipeline in willrun:
ppol = pipepol.get(pipeline)
launchPipeline(pipeline, runid,
masternode[pipeline], workingdir[pipeline],
envscript[pipeline])
# send input data events
for efile in exposureFiles:
if not os.path.isabs(efile):
if os.path.exists(efile) and not efile.startswith("."):
efile = os.path.join(".", efile)
elif os.path.exists(os.path.join(os.environ[pkgdirvar],
"exposureLists",efile)):
efile = os.path.join(os.environ[pkgdirvar],
"exposureLists",efile)
if not os.path.exists(efile):
logger.log(Log.WARN, "Exposure list file not found: " + efile)
logger.log(Log.DEBUG, "Pausing for 15s, waiting for setup...")
time.sleep(15)
logger.log(Log.INFO, "Sending exposure data from " + efile)
logger.log(Log.DEBUG,
"executing: %s < %s" % (eventgenerator, efile))
with file(efile) as expfile:
if subprocess.call(eventgenerator.split(), stdin=expfile) != 0:
raise RuntimeError("Failed to execute eventgenerator")
def testRead(self):
ps = PolicyString(self.data)
p = Policy.createPolicy(ps)
self.assertEquals(p.get("int"), 7)
self.assertEquals(p.get("dbl"), -1.0)
def testSimpleLoad(self):
# n = mwid.Citizen_census(0)
p = Policy.createPolicy(
os.path.join(proddir, "examples", "EventTransmitter_policy.paf"))
self.assertEqual(p.get("transmitter.serializationFormat"), "deluxe")
p = None
def testSimpleLoad(self):
# n = mwid.Citizen_census(0)
p = Policy.createPolicy("examples/EventTransmitter_policy.paf")
self.assertEquals(p.get("transmitter.serializationFormat"), "deluxe")
p = None