def testPolicyCopy(self):
p = Policy.createPolicy(os.path.join(proddir, "examples", "EventTransmitter_policy.paf"))
pp = Policy(p, True)
self.assertEqual(p.get("transmitter.serializationFormat"), "deluxe")
self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
p = None
self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
def testSetNothing(self):
p = Policy()
try:
p.set("foo", None)
self.assertTrue(False, "Setting value to None succeeded.")
except RuntimeError:
self.assertFalse(p.exists("foo"))
def setup(self):
deffile = DefaultPolicyFile("ctrl_sched","GetAJob_dict.paf","policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self,"policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
self.jobid = None
self.tagLogger(None)
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["jobIdentity"] = \
self.policy.getString("outputKeys.jobIdentity")
self.clipboardKeys["inputDatasets"] = \
self.policy.getString("outputKeys.inputDatasets")
self.clipboardKeys["outputDatasets"] = \
self.policy.getString("outputKeys.outputDatasets")
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("outputKeys.completedDatasets")
self.log.log(Log.INFO-1, "clipboard keys: " + str(self.clipboardKeys))
topic = self.policy.getString("pipelineEvent")
self.client = GetAJobClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), self.log)
self.log.log(Log.INFO-1,
"Using OriginatorId = %d" % self.client.getOriginatorId())
def setup(self):
deffile = DefaultPolicyFile("ctrl_sched", "GetAJob_dict.paf",
"policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self, "policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
self.jobid = None
self.tagLogger(None)
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["jobIdentity"] = \
self.policy.getString("outputKeys.jobIdentity")
self.clipboardKeys["inputDatasets"] = \
self.policy.getString("outputKeys.inputDatasets")
self.clipboardKeys["outputDatasets"] = \
self.policy.getString("outputKeys.outputDatasets")
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("outputKeys.completedDatasets")
self.log.log(Log.INFO - 1,
"clipboard keys: " + str(self.clipboardKeys))
topic = self.policy.getString("pipelineEvent")
self.client = GetAJobClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), self.log)
self.log.log(Log.INFO - 1,
"Using OriginatorId = %d" % self.client.getOriginatorId())
def testPolicyCopy(self):
p = Policy.createPolicy(os.path.join(proddir, "examples", "EventTransmitter_policy.paf"))
pp = Policy(p, True)
self.assertEqual(p.get("transmitter.serializationFormat"), "deluxe")
self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
p = None
self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
def testPolicyCopy(self):
p = Policy.createPolicy("examples/EventTransmitter_policy.paf")
pp = Policy(p, True)
self.assertEquals(p.get("transmitter.serializationFormat"), "deluxe")
self.assertEquals(pp.getString("transmitter.serializationFormat"), "deluxe")
p = None
self.assertEquals(pp.getString("transmitter.serializationFormat"), "deluxe")
def main(self, argv=None):
self.parseArgs(argv)
# 1. load policy
policy = self.tryThis(Policy,
"Reading policy file \"" + self.policyFile + "\"",
self.policyFile)
# resolve policy file references
polLoadDir = self.options.loadPolicy
polLoadDesc = polLoadDir
if polLoadDir is None:
if self.policyFile.find("/") >= 0:
polLoadDir = self.policyFile.rpartition("/")[0]
polLoadDesc = polLoadDir
else:
polLoadDir = ""
polLoadDesc = "current directory; " \
"try -l DIR or --load-policy-references=DIR"
if self.verbose:
print("No policy load dir specified; defaulting to " + polLoadDesc)
message = "Resolving references in " + self.policyFile \
+ ",\n using " + polLoadDesc
self.tryThis(policy.loadPolicyFiles, message, polLoadDir, True)
# derive short name by trimming off path & suffix
shortname = self.policyFile
if shortname.endswith(".paf"):
shortname = shortname.rpartition(".paf")[0]
if shortname.find("/") >= 0:
shortname = shortname.rpartition("/")[2]
if self.verbose:
print("Short name =", shortname)
# 2. create a dictionary from it
dictionary = Policy()
dictionary.set("target", shortname)
self.generateDict(policy, dictionary)
# TODO: remove commas from lists
if self.verbose:
print("Generating Dictionary for Policy " + self.policyFile + ":")
print("---------------------------Policy---------------------------")
print(policy)
print("-------------------------Dictionary-------------------------")
realDict = Dictionary(dictionary)
try:
realDict.validate(policy)
print("#<?cfg paf dictionary ?>")
writer = PAFWriter()
writer.write(dictionary)
print(writer.toString())
# print(dictionary)
except lsst.pex.exceptions.Exception as e:
ve = e.args[0]
print("Internal error: validation fails against extrapolated dictionary:")
print(ve.describe(" - "))
def testSetNothing(self):
p = Policy()
try:
p.set("foo", None)
self.assert_(False, "Setting value to None succeeded.")
except RuntimeError:
self.assertFalse(p.exists("foo"))
def testChildDef(self):
# simple
d = Dictionary(self.getTestDictionary("childdef_simple_dictionary.paf"))
p = Policy(self.getTestDictionary("childdef_simple_policy_good.paf"))
d.validate(p)
p = Policy(self.getTestDictionary("childdef_simple_policy_bad.paf"))
ve = ValidationError("Dictionary_1.py", 1, "testChildDef")
d.validate(p, ve.cpp)
self.assertEqual(ve.getErrors("baz"), ValidationError.WRONG_TYPE)
# multiple childDefs (DictionaryError)
d = Dictionary(self.getTestDictionary("childdef_dictionary_bad_multiple.paf"))
self.assertRaiseLCE(DictionaryError, "Multiple childDef",
d.validate, "Dictionary specifies unknown types", p)
# complex
d = Dictionary(self.getTestDictionary("childdef_complex_dictionary.paf"))
p = Policy(self.getTestDictionary("childdef_complex_policy_good_1.paf"))
d.validate(p)
p = Policy(self.getTestDictionary("childdef_complex_policy_good_2.paf"))
d.validate(p)
p = Policy(self.getTestDictionary("childdef_complex_policy_bad_1.paf"))
ve = ValidationError("Dictionary_1.py", 1, "testChildDef")
d.validate(p, ve.cpp)
self.assertEqual(ve.getErrors("joe"), ValidationError.NOT_AN_ARRAY)
self.assertEqual(ve.getErrors("deb"), ValidationError.NOT_AN_ARRAY)
self.assertEqual(ve.getErrors("bob"), ValidationError.NOT_AN_ARRAY)
self.assertEqual(ve.getErrors("bob.bar"), ValidationError.NOT_AN_ARRAY)
self.assertEqual(ve.getErrors("nested.helen.qux"),
ValidationError.MISSING_REQUIRED)
self.assertEqual(ve.getErrors("nested.marvin.rafael"),
ValidationError.TOO_MANY_VALUES)
self.assertEqual(ve.getErrors("disallowed.foo"),
ValidationError.TOO_MANY_VALUES)
self.assertEqual(ve.getParamCount(), 7)
def testProcessDataset(self):
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 0)
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
# manipulate the policy
idp = Policy.createPolicy(PolicyString(idpolicy))
spolicy.set("job.identity", idp)
self.sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
# pdb.set_trace()
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=15)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 1)
self.assertEquals(self.bb.queues.jobsPossible.length(), 1)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
self.assertEquals(self.sched.nameNumber, 2)
ds = Dataset("PostISR", visitid=95, ccdid=22, snapid=0, ampid=15)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 2)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(1)
self.assertEquals(job.getName(), "Job-2")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
inputs = job.getInputDatasets()
self.assertEquals(len(inputs), 16)
self.assertEquals(inputs[0].type, "PostISR")
self.assertEquals(self.sched.nameNumber, 3)
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=14)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 3)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 14)
# pdb.set_trace()
for i in xrange(14):
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=i)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 17)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
self.assert_(job.isReady())
def setup(self, policyDict="DataReady_dict.paf"):
deffile = DefaultPolicyFile("ctrl_sched", policyDict, "policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self,"policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("inputKeys.completedDatasets")
self.clipboardKeys["possibleDatasets"] = \
self.policy.getString("inputKeys.possibleDatasets")
self.dataclients = []
clpols = []
if self.policy.exists("datasets"):
clpols = self.policy.getPolicyArray("datasets")
for pol in clpols:
dstype = None
if pol.exists("datasetType"):
dstype = pol.getString("datasetType")
topic = pol.getString("dataReadyEvent")
reportAll = pol.getBool("reportAllPossible")
client = DataReadyClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), dstype,
reportAll)
self.dataclients.append(client)
def setup(self, policyDict="DataReady_dict.paf"):
deffile = DefaultPolicyFile("ctrl_sched", policyDict, "policies")
defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())
if not hasattr(self, "policy") or not self.policy:
self.policy = Policy()
self.policy.mergeDefaults(defpol.getDictionary())
# self.mode = self.policy.getString("mode")
# if self.mode not in "parallel serial":
# raise RuntimeError("Stage %s: Unsupported mode: %s" %
# (self.getName(), self.mode))
self.clipboardKeys = {}
self.clipboardKeys["completedDatasets"] = \
self.policy.getString("inputKeys.completedDatasets")
self.clipboardKeys["possibleDatasets"] = \
self.policy.getString("inputKeys.possibleDatasets")
self.dataclients = []
clpols = []
if self.policy.exists("datasets"):
clpols = self.policy.getPolicyArray("datasets")
for pol in clpols:
dstype = None
if pol.exists("datasetType"):
dstype = pol.getString("datasetType")
topic = pol.getString("dataReadyEvent")
reportAll = pol.getBool("reportAllPossible")
client = DataReadyClient(self.getRun(), self.getName(), topic,
self.getEventBrokerHost(), dstype,
reportAll)
self.dataclients.append(client)
def testFromPolicy(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
p = Policy()
p.set("type", type)
# pdb.set_trace()
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assert_(ds.path is None)
self.assert_(ds.ids is None)
p.set("path", path)
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is None)
p.set("ids.ccdid", ccdid)
p.set("ids.visitid", visitid)
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
class BigBoolTestCase(unittest.TestCase):
def setUp(self):
self.policy = Policy()
def tearDown(self):
del self.policy
def testBigBoolArray(self):
biglen = 1000
self.policy.isBool("true")
for i in range(biglen):
self.policy.add("true", True)
v = self.policy.getArray("true")
self.assertEqual(len(v), biglen, "wrong big number of values in array")
for i in range(biglen):
self.assertTrue(v[i], "big array with bad value")
del v
self.assertTrue(True, "Blew up True")
fd = open("/dev/null", "w")
print("look: %s" % True, file=fd)
fd.close()
def testTypos(self):
base = "pex_policy:tests/urn:indirect_parent_typo_"
self.assertRaiseLCE(lsst.pex.exceptions.IoError, "failure opening Policy file",
UrnPolicyFile(base + "1.paf").load, "Typo in URN",
Policy())
self.assertRaiseLCE(lsst.pex.exceptions.IoError, "failure opening Policy file",
UrnPolicyFile(base + "2.paf").load, "Typo in URN",
Policy())
def main():
"""
run the job office
"""
(cl.opts, cl.args) = cl.parse_args()
if len(cl.args) == 0:
fail("Missing policy file")
if not os.path.exists(cl.args[0]):
fail("%s: policy file not found" % cl.args[0])
if not os.path.exists(cl.opts.rootdir):
fail("%s: root directory not found" % cl.opts.rootdir)
if not cl.opts.runid:
logger.log(Log.WARN, "No RunID given (via -r)")
defpolf = DefaultPolicyFile("ctrl_sched", "JobOffice_dict.paf", "policies")
policy = Policy.createPolicy(cl.args[0])
policy.mergeDefaults(Policy.createPolicy(defpolf,
defpolf.getRepositoryPath()))
name = policy.getString("name")
# set job office root directory
if not os.path.isabs(cl.opts.rootdir):
cl.opts.rootdir = os.path.abspath(cl.opts.rootdir)
persistdir = os.path.join(cl.opts.rootdir, name)
if policy.exists("persist.dir"):
persistdir = policy.get("persist.dir") % \
{"schedroot": cl.opts.rootdir, "name": name }
# create the logger(s)
logfile = cl.opts.logfile
if not logfile:
logfile = os.path.join(persistdir, "joboffice.log")
if not os.path.exists(logfile):
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
if not cl.opts.asdaemon or cl.opts.toscreen:
ofclogger = DualLog(logfile, Log.DEBUG, Log.DEBUG, False)
# logging bug workaround
ofclogger.setScreenVerbose(False)
else:
ofclogger = Log()
ofclogger.addDestination(logfile)
ofclogger.setThreshold(run.verbosity2threshold(cl.opts.logverb, 0))
ofclogger.log(-2,"office threshold: %i" % ofclogger.getThreshold())
try:
# create the JobOffice instance
office = createJobOffice(cl.opts.rootdir, policy, ofclogger,
cl.opts.runid, cl.opts.brokerhost,
cl.opts.brokerport)
except Exception, ex:
logger.log(Log.FATAL, "Failed to create job office: " + str(ex))
raise
sys.exit(1)
def testBadDictionary(self):
d = Dictionary(self.getTestDictionary("dictionary_bad_policyfile.paf"))
self.assertRaiseLCE(DictionaryError, "Illegal type: \"PolicyFile\"",
d.makeDef("file_type").getType,
"Dictionary specified PolicyFile type")
d = Dictionary(self.getTestDictionary("dictionary_bad_unknown_type.paf"))
self.assertRaiseLCE(DictionaryError, "Unknown type: \"NotAType\"",
d.makeDef("something").getType,
"Dictionary specifies unknown types")
d = Dictionary(self.getTestDictionary("dictionary_bad_type_type.paf"))
self.assertRaiseLCE(DictionaryError, "Expected string",
d.makeDef("something").getType,
"Expected string \"type\" type")
self.assertRaiseLCE(DictionaryError, "property found at bad: min_occurs",
Dictionary,
"Dictionary has mispelled keyword \"min_occurs\".",
self.getTestDictionary("dictionary_bad_keyword.paf"))
dbmd = self.getTestDictionary("dictionary_bad_multiple_definitions.paf")
self.assertRaiseLCE(DictionaryError, "expected a single",
Dictionary,
"Dictionary has two 'definitions' sections",
dbmd)
p = Policy(self.getTestDictionary("values_policy_good_1.paf"))
d = Dictionary(self.getTestDictionary("dictionary_bad_multiple_min.paf"))
self.assertRaiseLCE(DictionaryError, "Min value for int_ra", d.validate,
"Two mins specified.", p)
d = Dictionary(self.getTestDictionary("dictionary_bad_multiple_max.paf"))
self.assertRaiseLCE(DictionaryError, "Max value for int_ra", d.validate,
"Two maxes specified.", p)
d = Dictionary(self.getTestDictionary("dictionary_bad_min_wrong_type.paf"))
self.assertRaiseLCE(DictionaryError,
"Wrong type for int_range_count_type min",
d.validate, "Wrong min type.", p)
d = Dictionary(self.getTestDictionary("dictionary_bad_max_wrong_type.paf"))
self.assertRaiseLCE(DictionaryError,
"Wrong type for int_range_count_type max",
d.validate, "Wrong max type.", p)
# conflict between minOccurs and maxOccurs
d = Dictionary(self.getTestDictionary("conflict_occurs_dictionary.paf"))
p = Policy(self.getTestDictionary("conflict_occurs_policy_1.paf"))
ve = ValidationError("Dictionary_1.py", 1, "testBadDictionary")
d.validate(p, ve.cpp)
self.assertEqual(ve.getErrors("1to0"), ValidationError.TOO_MANY_VALUES)
self.assertEqual(ve.getErrors("2to1"), ValidationError.NOT_AN_ARRAY)
self.assertEqual(ve.getParamCount(), 2)
p = Policy(self.getTestDictionary("conflict_occurs_policy_2.paf"))
ve = ValidationError("Dictionary_1.py", 1, "testBadDictionary")
d.validate(p, ve.cpp)
self.assertEqual(ve.getErrors("1to0"), ValidationError.MISSING_REQUIRED)
self.assertEqual(ve.getErrors("2to1"), ValidationError.TOO_MANY_VALUES)
self.assertEqual(ve.getParamCount(), 2)
def testFromPolicy(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
p = Policy()
p.set("type", type)
# pdb.set_trace()
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assert_(ds.path is None)
self.assert_(ds.ids is None)
p.set("path", path)
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is None)
p.set("ids.ccdid", ccdid)
p.set("ids.visitid", visitid)
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
def setUp(self):
p = Policy()
p.set("foo", "bar")
p.set("count", 3)
p.set("files", "goob")
p.add("files", "gurn")
impl = bb.PolicyBlackboardItem()
impl._props = p
self.bbi = bb.ImplBlackboardItem(impl)
self.initCount = 3
def __init__(self, policyfile=None):
"""
create an item with the given properties.
@param policyfile A policy
"""
# the properties attached to this items
self._props = None
if policyfile:
self._props = Policy.createPolicy(policyfile)
else:
self._props = Policy()
def __init__(self, policyfile=None):
"""
create an item with the given properties.
@param policyfile A policy
"""
# the properties attached to this items
self._props = None
if policyfile:
self._props = Policy.createPolicy(policyfile)
else:
self._props = Policy()
def testRepos(self):
# when the repository is mis-specified, local files cannot be loaded
upf = UrnPolicyFile("pex_policy:tests:urn/indirect_parent_good.paf")
# we expect it to look in <package>/tests/simple.paf
expectedFile = os.environ["PEX_POLICY_DIR"] + "/tests/simple.paf"
self.assertRaisesEx(IoErrorException,
"failure opening Policy file: " + expectedFile,
upf.load, "Wrong repository dir.", Policy())
# a PAF file designed to have "tests" as it repository
p = Policy()
UrnPolicyFile("pex_policy:tests:urn/local_tests_repos.paf").load(p)
self.assert_(p.get("local.polish") == "fancy")
def testTypeValidation(self):
d = Dictionary(self.getTestDictionary("types_dictionary.paf"))
self.assertEqual(
d.makeDef("undef_type").getType(), Policy.UNDEF,
"UNDEF definition type")
self.assertEqual(
d.makeDef("bool_type").getType(), Policy.BOOL,
"BOOL definition type")
self.assertEqual(
d.makeDef("int_type").getType(), Policy.INT, "INT definition type")
self.assertEqual(
d.makeDef("double_type").getType(), Policy.DOUBLE,
"DOUBLE definition type")
self.assertEqual(
d.makeDef("string_type").getType(), Policy.STRING,
"STRING definition type")
self.assertEqual(
d.makeDef("policy_type").getType(), Policy.POLICY,
"POLICY definition type")
self.assertEqual(
d.makeDef("file_type").getType(), Policy.POLICY,
"POLICY definition type (substituted for PolicyFile)")
p = Policy(self.getTestDictionary("types_policy_good.paf"))
ve = ValidationError("Dictionary_1.py", 0, "testTypeValidation")
d.validate(p, ve.cpp)
self.assertEqual(ve.getErrors("file_type"), ValidationError.NOT_LOADED,
"require loadPolicyFiles before validating")
self.assertEqual(ve.getErrors("undef_file"),
ValidationError.NOT_LOADED,
"require loadPolicyFiles before validating")
self.assertEqual(ve.getErrors(), ValidationError.NOT_LOADED,
"no other errors")
self.assertEqual(ve.getParamCount(), 2, "no other errors")
p.loadPolicyFiles(self.getTestDictionary(), True)
ve = ValidationError("Dictionary_1.py", 0, "testTypeValidation")
d.validate(p, ve.cpp)
self.assertEqual(ve.getErrors("undef_type"), 0, "no errors with undef")
self.assertEqual(ve.getErrors("int_type"), 0, "no errors with int")
self.assertEqual(ve.getErrors("double_type"), 0,
"no errors with double")
self.assertEqual(ve.getErrors("bool_type"), 0, "no errors with bool")
self.assertEqual(ve.getErrors("string_type"), 0,
"no errors with string")
self.assertEqual(ve.getErrors("policy_type"), 0,
"no errors with policy")
self.assertEqual(ve.getErrors("file_type"), 0, "no errors with file")
self.assertEqual(ve.getErrors(), 0, "no errors overall")
def testRepos(self):
# when the repository is mis-specified, local files cannot be loaded
upf = UrnPolicyFile("pex_policy:tests:urn/indirect_parent_good.paf")
# we expect it to look in <package>/tests/simple.paf
pexPolicyDir = lsst.utils.getPackageDir('pex_policy')
expectedFile = pexPolicyDir + "/tests/simple.paf"
self.assertRaiseLCE(lsst.pex.exceptions.IoError,
"failure opening Policy file: " + expectedFile,
upf.load, "Wrong repository dir.", Policy())
# a PAF file designed to have "tests" as it repository
p = Policy()
UrnPolicyFile("pex_policy:tests:urn/local_tests_repos.paf").load(p)
self.assertEqual(p.get("local.polish"), "fancy")
def testRepos(self):
# when the repository is mis-specified, local files cannot be loaded
upf = UrnPolicyFile("pex_policy:tests:urn/indirect_parent_good.paf")
# we expect it to look in <package>/tests/simple.paf
pexPolicyDir = lsst.utils.getPackageDir('pex_policy')
expectedFile = pexPolicyDir + "/tests/simple.paf"
self.assertRaiseLCE(lsst.pex.exceptions.IoError,
"failure opening Policy file: " + expectedFile,
upf.load, "Wrong repository dir.", Policy())
# a PAF file designed to have "tests" as it repository
p = Policy()
UrnPolicyFile("pex_policy:tests:urn/local_tests_repos.paf").load(p)
self.assertEqual(p.get("local.polish"), "fancy")
def setUp(self):
if not os.path.exists(self.propfile):
p = Policy()
p.set("foo", "bar")
p.set("count", 3)
p.set("files", "goob")
p.add("files", "gurn")
w = PAFWriter(self.propfile)
w.write(p, True)
w.close()
self.bbi = bb.PolicyBlackboardItem(self.propfile)
self.initCount = 3
def testPostprocess(self):
print "Events in Postprocess..."
pol = Policy()
pol.add("RunMode", "postprocess")
pol.add("keysToPublish", "loose=change")
pol.add("keysToPublish", "planned=obselesence")
self._runStageWithPolicy(pol)
def setUp(self):
p = Policy()
p.set("foo", "bar")
p.set("count", 3)
p.set("files", "goob")
p.add("files", "gurn")
impl = bb.PolicyBlackboardItem()
impl._props = p
self.bbi = bb.ImplBlackboardItem(impl)
self.initCount = 3
def testWrongType(self):
d = Dictionary(self.getTestDictionary("types_dictionary.paf"))
p = Policy(self.getTestDictionary("types_policy_bad_bool.paf"))
ve = ValidationError("Dictionary_1.py", 0, "testWrongType")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
self.assert_(ve.getParamCount() == 5, "number of errors")
self.assert_(ve.getErrors("bool_type") == 0, "correct type")
p = Policy(self.getTestDictionary("types_policy_bad_int.paf"))
ve = ValidationError("Dictionary_1.py", 1, "testWrongType")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
self.assert_(ve.getParamCount() == 5, "number of errors")
self.assert_(ve.getErrors("int_type") == 0, "correct type")
self.assert_(ve.getErrors("double_type") == ValidationError.WRONG_TYPE,
"int can't pass as double")
p = Policy(self.getTestDictionary("types_policy_bad_double.paf"))
ve = ValidationError("Dictionary_1.py", 2, "testWrongType")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
self.assert_(ve.getParamCount() == 5, "number of errors")
self.assert_(ve.getErrors("double_type") == 0, "correct type")
p = Policy(self.getTestDictionary("types_policy_bad_string.paf"))
ve = ValidationError("Dictionary_1.py", 3, "testWrongType")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
self.assert_(ve.getParamCount() == 5, "number of errors")
self.assert_(ve.getErrors("string_type") == 0, "correct type")
p = Policy(self.getTestDictionary("types_policy_bad_policy.paf"))
ve = ValidationError("Dictionary_1.py", 4, "testWrongType")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
self.assert_(ve.getParamCount() == 4, "number of errors")
self.assert_(ve.getErrors("policy_type") == 0, "correct type")
self.assert_(ve.getErrors("file_type") == 0, "correct type")
p = Policy(self.getTestDictionary("types_policy_bad_file.paf"))
ve = ValidationError("Dictionary_1.py", 5, "testWrongType")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors() == ValidationError.NOT_LOADED, "not loaded")
self.assert_(ve.getParamCount() == 6, "number of errors")
self.assert_(ve.getErrors("bool_type") == ValidationError.NOT_LOADED,
"not loaded")
self.assert_(ve.getErrors("file_type") == ValidationError.NOT_LOADED,
"not loaded")
self.assert_(ve.getErrors("policy_type") == ValidationError.NOT_LOADED,
"not loaded")
p.loadPolicyFiles(self.getTestDictionary(), True)
ve = ValidationError("Dictionary_1.py", 6, "testWrongType")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
self.assert_(ve.getErrors("file_type") == 0, "correct type")
self.assert_(ve.getErrors("policy_type") == 0, "correct type")
self.assert_(ve.getParamCount() == 4, "wrong type")
def testIndirect(self):
urn = "@urn:eupspkg:pex_policy:tests/urn:indirect_parent_good.paf"
p = Policy(urn)
self.assertEqual(p.get("urn_full.name"), "Simple Policy")
self.assertEqual(p.get("urn_brief.name"), "Simple Policy")
self.assertEqual(p.get("urn_mixed_case.name"), "Simple Policy")
self.assertEqual(p.get("local.foo"), "bar")
p = Policy()
UrnPolicyFile("pex_policy:tests/urn:level_1.paf").load(p)
self.assertEqual(p.get("foo.bar.baz.qux.quux"), "schmazzle")
def testReference(self):
addr = "pex_policy:examples:EventTransmitter_policy.paf"
p = Policy()
p.set("transmitter.logVerbosity", "not")
UrnPolicyFile(addr).load(p)
self.assertEqual(p.get("transmitter.logVerbosity"), "debug")
p.set("transmitter.logVerbosity", "not")
UrnPolicyFile("urn:eupspkg:" + addr).load(p)
self.assertEqual(p.get("transmitter.logVerbosity"), "debug")
p.set("transmitter.logVerbosity", "not")
UrnPolicyFile("@@" + addr).load(p)
self.assertEqual(p.get("transmitter.logVerbosity"), "debug")
def main():
try:
(cl.opts, cl.args) = cl.parse_args();
setVerbosity(cl.opts.verbosity)
nodes = []
if cl.opts.policy is not None:
policy = Policy.createPolicy(cl.opts.policy)
nodes = getHeadNodes(policy)
nodes.extend(cl.args)
logger.log(Log.DEBUG, "Killing pipelines on " + ", ".join(nodes))
remcmd = "%s %s" % \
(os.path.join(os.environ[pkgdirvar], "bin", remkill),cl.opts.runid)
remcmd = remcmd.strip()
for node in nodes:
cmd = ("ssh", node, remcmd)
logger.log(Log.INFO, "executing: %s %s '%s'" % cmd)
if subprocess.call(cmd) != 0:
logger.log(Log.WARN, "Failed to kill processes on " + node)
except:
tb = traceback.format_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
logger.log(Log.FATAL, tb[-1].strip())
logger.log(Log.DEBUG, "".join(tb[0:-1]).strip())
sys.exit(1)
sys.exit(0)
def extractIncludedFilenames(prodPolicyFile, repository=".",
pipefile=None, logger=None):
"""
extract all the filenames included, directly or indirectly, from the
given policy file. When a repository is provided, included files will
be recursively opened and searched. The paths in the returned set
will not include the repository directory. Use pipefile to skip
the inclusion of pipeline policy files.
@param prodPolicyFile the policy file to examine. This must
be the full path to the file
@param repository the policy repository. If None, the current
directory is assumed.
@param pipefile the hierarchical policy name for a pipeline
definition. Any included policy filenames
at this node or lower will not be added.
@param logger if provided, use this Log to record any
warnings about missing or bad files;
otherwise, problems are silently ignored.
@return set containing the unique set of policy filenames found,
including the given top file and the
"""
prodPolicy = Policy.createPolicy(prodPolicyFile, False)
filenames = set([prodPolicyFile])
ProvenanceSetup._listFilenames(filenames, prodPolicy, None, repository,
pipefile, logger)
filenames.discard(prodPolicyFile)
return filenames
def testSetAdd(self):
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_good.paf"),
"", True)
self.assert_(p.canValidate())
self.assertValidationError(
ValidationError.TOO_MANY_VALUES,
p.add, "bool_set_count", True)
self.assert_(p.valueCount("bool_set_count") == 1)
self.assertValidationError(
ValidationError.VALUE_DISALLOWED,
p.set, "bool_set_count", False)
self.assert_(p.getBool("bool_set_count") == True)
p.set("int_range_count", -7)
self.assertValidationError(
ValidationError.VALUE_OUT_OF_RANGE,
p.add, "int_range_count", 10)
# add & set don't check against minOccurs, but validate() does
try:
p.validate()
except ValidationError as ve:
self.assert_(ve.getErrors("int_range_count")
== ValidationError.NOT_AN_ARRAY)
self.assert_(ve.getErrors("required")
== ValidationError.MISSING_REQUIRED)
p.add("int_range_count", -8)
p.set("required", "foo")
p.validate()
def testCreateName(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
ds = Dataset("PostISR", ampid=3)
self.assertEquals(sched.createName(ds), "Job-1")
def testSimpleValidate(self):
d = Dictionary(self.getTestDictionary("simple_dictionary.paf"))
p = Policy(self.getTestDictionary("simple_policy.paf"))
ve = ValidationError("Dictionary_1.py", 0, "testSimpleValidate")
d.validate(p, ve.cpp)
self.assertEqual(ve.getErrors("name"), 0, "no errors in name")
self.assertEqual(ve.getErrors("height"), 0, "no errors in height")
self.assertEqual(ve.getErrors(), 0, "no errors overall")
def setUp(self):
if not os.path.exists(self.propfile):
p = Policy()
p.set("foo", "bar")
p.set("count", 3)
p.set("files", "goob")
p.add("files", "gurn")
w = PAFWriter(self.propfile)
w.write(p, True)
w.close()
self.bbi = bb.PolicyBlackboardItem(self.propfile)
self.initCount = 3
def __init__(self, ampPolicy="cfhtAmpBBoxPolicy.paf"):
CcdInfo.__init__(self) # Is this the best way to do this?
self.nCcd = 36 # Number of CCDs (0-indexed)
self.nAmp = 8 # Number of amplifiers (0-indexed)
ampBBoxDb = Policy(ampPolicy)
for amp in self.ampList():
p = ampBBoxDb.get("CcdBBox.Amp%d" % amp)
self.ampBBox[amp] = afwGeom.Box2I(afwGeom.Point2I(p.get("x0"), p.get("y0")),
afwGeom.Extent2I(p.get("width"), p.get("height")))
for amp in self.ampList(): # Build by hand
bbox = self.getAmpBBox(amp)
bbox.shift(32, 0) # Allow for overclock
self.trimSecBBox[amp] = bbox
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not StringIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "StringIDFilter_dict.paf",
"policies")
StringIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if StringIDFilter._dictionary:
p.mergeDefaults(StringIDFilter._dictionary)
name = "unknown"
vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("value"): vals = policy.getArray("value")
return StringIDFilter(name, vals)
def testCreateName2(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
spolicy.set("job.name.template", "%(type)s-v%(ampid)s")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
ds = Dataset("PostISR", ampid=3)
self.assertEquals(sched.createName(ds), "PostISR-v3")
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not StringIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "StringIDFilter_dict.paf",
"policies")
StringIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if StringIDFilter._dictionary:
p.mergeDefaults(StringIDFilter._dictionary)
name = "unknown"
vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("value"): vals = policy.getArray("value")
return StringIDFilter(name, vals)
class PolicyOutStringTestCase(unittest.TestCase):
def setUp(self):
self.policy = Policy()
self.policy.set("answer", 42)
self.policy.set("name", "ray")
def tearDown(self):
del self.policy
def testDest(self):
dest = PolicyStringDestination("#<?cfg paf policy ?>")
self.assertEqual(dest.getData(), "#<?cfg paf policy ?>")
def testWrite(self):
writer = PAFWriter()
writer.write(self.policy, True)
out = writer.toString()
self.assertTrue(out.startswith("#<?cfg paf policy ?>"))
def setUp(self):
policy = Policy.createPolicy(policyFile)
# pdb.set_trace()
self.joboffice = DataTriggeredJobOffice(testdir,
policy=policy,
log=None,
runId="testing",
brokerHost=brokerhost)
self.joboffice.log.setThreshold(self.joboffice.log.WARN)
self.jodir = os.path.join(testdir, "ccdassembly")
def __init__(self, ampPolicy="cfhtAmpBBoxPolicy.paf"):
CcdInfo.__init__(self) # Is this the best way to do this?
self.nCcd = 36 # Number of CCDs (0-indexed)
self.nAmp = 8 # Number of amplifiers (0-indexed)
ampBBoxDb = Policy(ampPolicy)
for amp in self.ampList():
p = ampBBoxDb.get("CcdBBox.Amp%d" % amp)
self.ampBBox[amp] = afwGeom.Box2I(
afwGeom.Point2I(p.get("x0"), p.get("y0")),
afwGeom.Extent2I(p.get("width"), p.get("height")))
for amp in self.ampList(): # Build by hand
bbox = self.getAmpBBox(amp)
bbox.shift(32, 0) # Allow for overclock
self.trimSecBBox[amp] = bbox
class PolicyOutStringTestCase(unittest.TestCase):
def setUp(self):
self.policy = Policy()
self.policy.set("answer", 42)
self.policy.set("name", "ray")
def tearDown(self):
pass
def testDest(self):
dest = PolicyStringDestination("#<?cfg paf policy ?>")
self.assertEquals(dest.getData(), "#<?cfg paf policy ?>")
def testWrite(self):
writer = PAFWriter()
writer.write(self.policy, True)
out = writer.toString();
self.assert_(out.startswith("#<?cfg paf policy ?>"))
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not IntegerIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "IntegerIDFilter_dict.paf",
"policies")
IntegerIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if IntegerIDFilter._dictionary:
p.mergeDefaults(IntegerIDFilter._dictionary)
name = "unknown"
min = lim = vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("min"): min = policy.getInt("min")
if policy.exists("lim"): lim = policy.getInt("lim")
if policy.exists("value"): vals = policy.getArray("value")
return IntegerIDFilter(name, min, lim, vals)
def testCtor(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
self.assert_(sched.nametmpl is None)
self.assertEquals(sched.defaultName, "Job")
self.assertEquals(sched.nameNumber, 1)
self.assertEquals(len(sched.triggers), 1)
self.assertEquals(len(sched.inputdata), 1)
def fromPolicy(policy):
"""
create an IntegerIDFilter from an "id" policy
"""
if not IntegerIDFilter._dictionary:
pdf = DefaultPolicyFile("ctrl_sched", "IntegerIDFilter_dict.paf",
"policies")
IntegerIDFilter._dictionary = Policy.createPolicy(pdf)
p = Policy(policy, True)
if IntegerIDFilter._dictionary:
p.mergeDefaults(IntegerIDFilter._dictionary)
name = "unknown"
min = lim = vals = None
if policy.exists("name"): name = policy.getString("name")
if policy.exists("min"): min = policy.getInt("min")
if policy.exists("lim"): lim = policy.getInt("lim")
if policy.exists("value"): vals = policy.getArray("value")
return IntegerIDFilter(name, min, lim, vals)
def testTypeValidation(self):
d = Dictionary(self.getTestDictionary("types_dictionary.paf"))
self.assert_(d.makeDef("undef_type") .getType() == Policy.UNDEF,
"UNDEF definition type")
self.assert_(d.makeDef("bool_type") .getType() == Policy.BOOL,
"BOOL definition type")
self.assert_(d.makeDef("int_type") .getType() == Policy.INT,
"INT definition type")
self.assert_(d.makeDef("double_type").getType() == Policy.DOUBLE,
"DOUBLE definition type")
self.assert_(d.makeDef("string_type").getType() == Policy.STRING,
"STRING definition type")
self.assert_(d.makeDef("policy_type").getType() == Policy.POLICY,
"POLICY definition type")
self.assert_(d.makeDef("file_type").getType() == Policy.POLICY,
"POLICY definition type (substituted for PolicyFile)")
p = Policy(self.getTestDictionary("types_policy_good.paf"))
ve = ValidationError("Dictionary_1.py", 0, "testTypeValidation")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors("file_type") == ValidationError.NOT_LOADED,
"require loadPolicyFiles before validating")
self.assert_(ve.getErrors("undef_file") == ValidationError.NOT_LOADED,
"require loadPolicyFiles before validating")
self.assert_(ve.getErrors() == ValidationError.NOT_LOADED, "no other errors")
self.assert_(ve.getParamCount() == 2, "no other errors")
p.loadPolicyFiles(self.getTestDictionary(), True)
ve = ValidationError("Dictionary_1.py", 0, "testTypeValidation")
d.validate(p, ve.cpp)
self.assert_(ve.getErrors("undef_type") == 0, "no errors with undef")
self.assert_(ve.getErrors("int_type") == 0, "no errors with int")
self.assert_(ve.getErrors("double_type") == 0, "no errors with double")
self.assert_(ve.getErrors("bool_type") == 0, "no errors with bool")
self.assert_(ve.getErrors("string_type") == 0, "no errors with string")
self.assert_(ve.getErrors("policy_type") == 0, "no errors with policy")
self.assert_(ve.getErrors("file_type") == 0, "no errors with file")
self.assert_(ve.getErrors() == 0, "no errors overall")
def testDefaults(self):
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_good.paf"),
"", True)
self.assertEqual(p.valueCount("bool_set_count"), 1)
self.assertIs(p.getBool("bool_set_count"), True)
self.assertEqual(p.valueCount("int_range_count"), 3)
self.assertEqual(p.getDouble("deep.sub_double"), 1.)
self.assertEqual(p.get("indirect4.string_type"), "foo")
try:
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_bad_1.paf"),
"", True)
except ValidationError as ve:
self.assertEqual(ve.getErrors("double"),
ValidationError.WRONG_TYPE)
self.assertEqual(ve.getErrors("int_range_count"),
ValidationError.NOT_AN_ARRAY)
self.assertEqual(ve.getErrors("bool_set_count"),
ValidationError.TOO_MANY_VALUES)
self.assertEqual(ve.getErrors("deep.sub_double"),
ValidationError.WRONG_TYPE)
self.assertEqual(ve.getParamCount(), 4)
def testDefaults(self):
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_good.paf"),
"", True)
self.assert_(p.valueCount("bool_set_count") == 1)
self.assert_(p.getBool("bool_set_count") == True)
self.assert_(p.valueCount("int_range_count") == 3)
self.assert_(p.getDouble("deep.sub_double") == 1.)
self.assert_(p.get("indirect4.string_type") == "foo")
try:
p = Policy.createPolicy(self.getTestDictionary("defaults_dictionary_bad_1.paf"),
"", True)
except ValidationError as ve:
self.assert_(ve.getErrors("double")
== ValidationError.WRONG_TYPE)
self.assert_(ve.getErrors("int_range_count")
== ValidationError.NOT_AN_ARRAY)
self.assert_(ve.getErrors("bool_set_count")
== ValidationError.TOO_MANY_VALUES)
self.assert_(ve.getErrors("deep.sub_double")
== ValidationError.WRONG_TYPE)
self.assert_(ve.getParamCount() == 4)
def testIndirect(self):
urn = "@urn:eupspkg:pex_policy:tests/urn:indirect_parent_good.paf"
p = Policy(urn)
self.assertEqual(p.get("urn_full.name"), "Simple Policy")
self.assertEqual(p.get("urn_brief.name"), "Simple Policy")
self.assertEqual(p.get("urn_mixed_case.name"), "Simple Policy")
self.assertEqual(p.get("local.foo"), "bar")
p = Policy()
UrnPolicyFile("pex_policy:tests/urn:level_1.paf").load(p)
self.assertEqual(p.get("foo.bar.baz.qux.quux"), "schmazzle")
