class TestCase(unittest.TestCase):
def setUp(self):
self.file = "tests/testFileLog-out.txt"
self.root = Log()
self.logger = Log(self.root, "test")
def tearDown(self):
if os.path.exists(self.file):
os.remove(self.file)
def testAddFile(self):
self.logger.addDestination(self.file)
self.logger.log(Log.INFO, "in file")
self.root.log(Log.INFO, "in bit bucket")
fd = open(self.file)
try:
lines = fd.readlines()
self.assertEquals(len(filter(lambda l: l.find("in file")>=0,lines)), 1)
self.assertEquals(len(filter(lambda l: l.find("bucket")>=0,lines)), 0)
finally:
fd.close()
def testPassAll(self):
self.logger.addDestination(self.file)
self.logger.log(Log.DEBUG, "in file")
self.logger.setThreshold(Log.DEBUG)
self.logger.log(Log.DEBUG, "debugging")
fd = open(self.file)
try:
lines = fd.readlines()
self.assertEquals(len(filter(lambda l: l.find("in file")>=0,lines)), 0)
self.assertEquals(len(filter(lambda l: l.find("debugging")>=0,lines)), 1)
finally:
fd.close()
def testVerbose(self):
self.logger.addDestination(self.file, True)
self.logger.log(Log.DEBUG, "in file")
self.logger.setThreshold(Log.DEBUG)
self.logger.log(Log.DEBUG, "debugging")
fd = open(self.file)
try:
lines = fd.readlines()
self.assertEquals(len(filter(lambda l: l.find("in file")>=0,lines)), 0)
self.assertEquals(len(filter(lambda l: l.find("debugging")>=0,lines)), 1)
self.assertEquals(len(filter(lambda l: l.find("LEVEL")>=0,lines)), 1)
finally:
fd.close()
def testGoodSubclass(self):
logger = Log(Log.getDefaultLog(), "tester")
logger.setThreshold(Log.FATAL+10)
recorder = GoodRecorder(self, logger)
recorder.record("goober.paf")
def testBadSubclass(self):
logger = Log(Log.getDefaultLog(), "tester")
logger.setThreshold(Log.FATAL+10)
recorder = BadRecorder(logger)
self.assertRaises(RuntimeError, recorder.record, "file")
# see <http://www.lsstcorp.org/LegalNotices/>.
#
"""
Tests of the ProvenanceRecorder
"""
from __future__ import print_function
import os
import unittest
import lsst.utils.tests
from lsst.pex.logging import Log
from lsst.ctrl.provenance import ProvenanceRecorder
from lsst.ctrl.provenance import ProvenanceSetup
logger = Log(Log.getDefaultLog(), "tester")
logger.setThreshold(Log.FATAL + 10)
repos = os.path.join(os.environ['CTRL_PROVENANCE_DIR'], "tests", "policy")
prodPolicyFile = os.path.join(repos, "production.paf")
class GoodRecorder(ProvenanceRecorder):
def __init__(self, testcase, logger=None):
ProvenanceRecorder.__init__(self, logger, True)
self.tester = testcase
def record(self, filename):
self.tester.assert_(len(filename) > 0, "empty filename found")
self._logger.log(Log.INFO, "recording " + filename)
self.tester.recorded += 1
def testGoodSubclass(self):
logger = Log(Log.getDefaultLog(), "tester")
logger.setThreshold(Log.FATAL + 10)
recorder = GoodRecorder(self, logger)
recorder.record("goober.paf")
def testBadSubclass(self):
logger = Log(Log.getDefaultLog(), "tester")
logger.setThreshold(Log.FATAL + 10)
recorder = BadRecorder(logger)
self.assertRaises(RuntimeError, recorder.record, "file")
class DataTriggeredSchedulerTestCase(unittest.TestCase):
def setUp(self):
self.bb = Blackboard(bbdir)
self.sched = None
self.logger = Log(rootlogger, "sched")
def tearDown(self):
self.logger.setThreshold(Log.INHERIT_THRESHOLD)
if os.path.exists(bbdir):
os.system("rm -rf %s" % bbdir)
self.sched = None
def testCtor(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
self.assert_(sched.nametmpl is None)
self.assertEquals(sched.defaultName, "Job")
self.assertEquals(sched.nameNumber, 1)
self.assertEquals(len(sched.triggers), 1)
self.assertEquals(len(sched.inputdata), 1)
def testCreateName(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
ds = Dataset("PostISR", ampid=3)
self.assertEquals(sched.createName(ds), "Job-1")
def testCreateName2(self):
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
spolicy.set("job.name.template", "%(type)s-v%(ampid)s")
sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
ds = Dataset("PostISR", ampid=3)
self.assertEquals(sched.createName(ds), "PostISR-v3")
def testProcessDataset(self):
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 0)
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
self.sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
# pdb.set_trace()
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=15)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 1)
self.assertEquals(self.bb.queues.jobsPossible.length(), 1)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
self.assertEquals(self.sched.nameNumber, 2)
ds = Dataset("PostISR", visitid=95, ccdid=22, snapid=0, ampid=15)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 2)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(1)
self.assertEquals(job.getName(), "Job-2")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
inputs = job.getInputDatasets()
self.assertEquals(len(inputs), 16)
self.assertEquals(inputs[0].type, "PostISR")
self.assertEquals(self.sched.nameNumber, 3)
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=14)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 3)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 14)
# pdb.set_trace()
for i in xrange(14):
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=i)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 17)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
self.assert_(job.isReady())
def testMakeAvail(self):
self.testProcessDataset()
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 17)
self.assertEquals(self.bb.queues.jobsAvailable.length(), 0)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assert_(job.isReady())
job = self.bb.queues.jobsPossible.get(1)
self.assertEquals(job.getName(), "Job-2")
self.assert_(not job.isReady())
self.sched.makeJobsAvailable()
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 17)
self.assertEquals(self.bb.queues.jobsAvailable.length(), 1)
self.assertEquals(self.bb.queues.jobsPossible.length(), 1)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-2")
self.assert_(not job.isReady())
job = self.bb.queues.jobsAvailable.get(0)
self.assertEquals(job.getName(), "Job-1")
class DataTriggeredSchedulerTestCase2(unittest.TestCase):
"""
test for a specific (unticketed) coding bug in _determineJobIdentity().
"""
def setUp(self):
self.bb = Blackboard(bbdir)
self.sched = None
self.logger = Log(rootlogger, "sched")
def tearDown(self):
self.logger.setThreshold(Log.INHERIT_THRESHOLD)
if os.path.exists(bbdir):
os.system("rm -rf %s" % bbdir)
self.sched = None
def testProcessDataset(self):
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 0)
policy = Policy.createPolicy(os.path.join(exampledir,
"ccdassembly-joboffice.paf"))
spolicy = policy.getPolicy("schedule")
# manipulate the policy
idp = Policy.createPolicy(PolicyString(idpolicy))
spolicy.set("job.identity", idp)
self.sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)
# pdb.set_trace()
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=15)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 1)
self.assertEquals(self.bb.queues.jobsPossible.length(), 1)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
self.assertEquals(self.sched.nameNumber, 2)
ds = Dataset("PostISR", visitid=95, ccdid=22, snapid=0, ampid=15)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 2)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(1)
self.assertEquals(job.getName(), "Job-2")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
inputs = job.getInputDatasets()
self.assertEquals(len(inputs), 16)
self.assertEquals(inputs[0].type, "PostISR")
self.assertEquals(self.sched.nameNumber, 3)
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=14)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 3)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 14)
# pdb.set_trace()
for i in xrange(14):
ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=i)
self.sched.processDataset(ds)
with self.bb.queues:
self.assertEquals(self.bb.queues.dataAvailable.length(), 17)
self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
job = self.bb.queues.jobsPossible.get(0)
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
self.assert_(job.isReady())
class TestCase(unittest.TestCase):
def setUp(self):
self.file = "tests/testFileLog-out.txt"
self.root = Log()
self.logger = Log(self.root, "test")
def tearDown(self):
del self.root
del self.logger
if os.path.exists(self.file):
os.remove(self.file)
def testAddFile(self):
self.logger.addDestination(self.file)
self.logger.log(Log.INFO, "in file")
self.root.log(Log.INFO, "in bit bucket")
fd = open(self.file)
try:
lines = fd.readlines()
self.assertEqual(len([l for l in lines if l.find("in file") >= 0]),
1)
self.assertEqual(len([l for l in lines if l.find("bucket") >= 0]),
0)
finally:
fd.close()
def testPassAll(self):
self.logger.addDestination(self.file)
self.logger.log(Log.DEBUG, "in file")
self.logger.setThreshold(Log.DEBUG)
self.logger.log(Log.DEBUG, "debugging")
fd = open(self.file)
try:
lines = fd.readlines()
self.assertEqual(len([l for l in lines if l.find("in file") >= 0]),
0)
self.assertEqual(
len([l for l in lines if l.find("debugging") >= 0]), 1)
finally:
fd.close()
def testVerbose(self):
self.logger.addDestination(self.file, True)
self.logger.log(Log.DEBUG, "in file")
self.logger.setThreshold(Log.DEBUG)
self.logger.log(Log.DEBUG, "debugging")
fd = open(self.file)
try:
lines = fd.readlines()
self.assertEqual(len([l for l in lines if l.find("in file") >= 0]),
0)
self.assertEqual(
len([l for l in lines if l.find("debugging") >= 0]), 1)
self.assertEqual(len([l for l in lines if l.find("LEVEL") >= 0]),
1)
finally:
fd.close()
"""
Tests of the ProvenanceRecorder
"""
import pdb # we may want to say pdb.set_trace()
import os
import sys
import unittest
import time
from lsst.pex.logging import Log
from lsst.ctrl.provenance import ProvenanceRecorder
from lsst.ctrl.provenance import ProvenanceSetup
logger = Log(Log.getDefaultLog(), "tester")
logger.setThreshold(Log.FATAL+10)
repos = os.path.join(os.environ['CTRL_PROVENANCE_DIR'], "tests", "policy")
prodPolicyFile = os.path.join(repos, "production.paf")
class GoodRecorder(ProvenanceRecorder):
def __init__(self, testcase, logger=None):
ProvenanceRecorder.__init__(self, logger, True)
self.tester = testcase
def record(self, filename):
self.tester.assert_(len(filename) > 0, "empty filename found")
self._logger.log(Log.INFO, "recording "+filename)
self.tester.recorded += 1
class ProvenanceSetupTestCase(unittest.TestCase):
