def testCheckAllWatchdogMax(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
dr.watchdog = MockWatchdog()
dr.watchdog.inProg = True
dr.watchdog.done = True
DAQRun.unHealthyCount = DAQRun.MAX_UNHEALTHY_COUNT
expCnt = 0
rtnVal = dr.check_all()
self.failIf(rtnVal, 'Expected call to succeed')
self.failUnless(dr.watchdog.threadCleared, 'Should have cleared thread')
self.failIf(dr.watchdog.watchStarted,
'Should not have started watchdog')
self.assertEquals(expCnt, DAQRun.unHealthyCount,
'UnhealthyCount should be %d, not %d' %
(expCnt, DAQRun.unHealthyCount))
logger.checkStatus(10)
def testWaitForRequiredBad(self):
comps = [(0, 'abc', 1, 'xxx', 1, 2),
(1, 'def', 2, 'yyy', 3, 4),
(2, 'ghi', 3, 'zzz', 5, 6)]
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
cnc.setComponents(comps[1:])
expRunNum = 100
expId = 123
dr.runStats.runNum = expRunNum
dr.runSetID = expId
DAQRun.COMP_TOUT = 0
required = []
for c in comps:
required.append('%s#%d' % (c[1], c[2]))
logger.addExpectedExact(('Starting run %d (waiting for required %d' +
' components to register w/ CnCServer)') %
(dr.runStats.runNum, len(required)))
try:
dr.build_run_set(cnc, required)
self.fail('Unexpected success')
except Exception, e:
self.assertEquals('Still waiting for ' + required[0], str(e),
'Unexpected exception message "%s"' % str(e))
def testBuildMissingOneOutput(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('aaa')
comp.addInput('xxx', 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('aaa', 456)
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
self.assertRaises(ValueError, mgr.makeRunset, nameList, logger)
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def __runSubrun(self,
compList,
runNum,
spadeDir="/tmp",
copyDir=None,
expectError=None):
logger = MockLogger('LOG')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runConfig = FakeRunConfig("XXXrunSubXXX")
clusterName = "cluster-foo"
runset = MyRunSet(MyParent(), runConfig, compList, logger)
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
runset.configure()
expState = "ready"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logDir = "/tmp"
#runset.startLogging(logDir, 123, clusterName)
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedRegexp("Could not stop run: .*")
try:
stopErr = runset.stopRun()
except RunSetException, ve:
if not "is not running" in str(ve):
raise ve
stopErr = False
def testVersion(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
self.assertEqual(live.version(),
MockCnC.RELEASE + "_" + MockCnC.REPO_REV)
log.checkStatus(1)
def testCheckAllNone(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
rtnVal = dr.check_all()
self.failUnless(rtnVal, 'Expected call to succeed')
logger.checkStatus(10)
def testBuildMultiInput(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('conn')
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('conn', 123)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addInput('conn', 456)
compList.append(comp)
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp("Built runset #\d+: .*")
runset = mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.assertEqual(mgr.numComponents(), 0)
found = mgr.findRunset(runset.id())
self.failIf(found is None, "Couldn't find runset #%d" % runset.id())
mgr.returnRunset(runset, logger)
self.assertEqual(mgr.numComponents(), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(mgr.numComponents(), 0)
logger.checkStatus(10)
def testRPCStopRunSuccess(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
dr.runState = 'RUNNING'
dr.rpc_stop_run()
self.assertEquals('STOPPING', dr.runState, 'Should be stopping, not ' +
dr.runState)
logger.checkStatus(10)
def testStartingNoStateArgs(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
self.assertRaisesMsg(LiveException("No stateArgs specified"),
live.starting, None)
def testRunningNothing(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
exc = LiveException("Cannot check run state; no active runset")
self.assertRaisesMsg(exc, live.running)
def testStoppingNoRunset(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
exc = LiveException("Cannot stop run; no active runset")
self.assertRaisesMsg(exc, live.stopping)
def testBuildMultiMissing(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addInput('xxx', 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addOutput('xxx')
compList.append(comp)
comp = MockComponent('feeHub', 0)
comp.addInput('xxx', 456)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addOutput('xxx')
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
try:
mgr.makeRunset(nameList, logger)
self.fail('Unexpected success')
except ValueError:
pass
except:
self.fail('Unexpected exception')
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def testBuildMultiInput(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('conn')
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('conn', 123)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addInput('conn', 456)
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
runset = mgr.makeRunset(nameList, logger)
self.assertEqual(len(mgr.pool), 0)
found = mgr.findRunset(runset.id)
self.failIf(found is None, "Couldn't find runset #" + str(runset.id))
mgr.returnRunset(runset)
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def testGetEventCounts(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
numEvts = 17
numMoni = 222
numSN = 51
numTCal = 93
dr.moni = MockMoni()
dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts))
dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni))
dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN))
dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal))
expId = 99
expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678),
(17, 'secondaryBuilders', 0, 'x', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
expCnts = (numEvts, numMoni, numSN, numTCal)
cnts = dr.getEventCounts()
self.assertEquals(len(expCnts), len(cnts),
'Expected %d event counts, not %d' %
(len(expCnts), len(cnts)))
for i in range(0, len(expCnts)):
self.assertEquals(expCnts[i], cnts[i],
'Expected event count #%d to be %d, not %d' %
(i, expCnts[i], cnts[i]))
logger.checkStatus(10)
def testRecovering(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
runSet = MockRunSet(runCfg)
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
cnc.setRunSet(runSet)
state = { "runConfig": runCfg, "runNumber": runNum }
self.failUnless(live.starting(state), "starting failed")
runSet.setExpectedStopError()
log.addExpectedExact("DAQLive recovered %s" % runSet)
self.failUnless(live.recovering(), "recovering failed")
def testRestartFailCluCfg(self):
compList = self.__buildCompList(
("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful"))
runConfig = FakeRunConfig("XXXrunCfgXXX")
logger = MockLogger('foo#0')
runset = MyRunSet(MyParent(), runConfig, compList, logger)
baseName = "failCluCfg"
clusterCfg = self.__buildClusterConfig(compList[1:], baseName)
logger.addExpectedExact(
("Cannot restart component %s: Not found" +
" in cluster config \"%s\"") %
(compList[0].fullName(), clusterCfg.configName()))
errMsg = None
for c in compList[1:]:
if errMsg is None:
errMsg = "Cycling components [" + c.fullName()
else:
errMsg += ", " + c.fullName()
if errMsg is not None:
errMsg += "]"
logger.addExpectedExact(errMsg)
runset.restartComponents(compList[:], clusterCfg, None, None, None,
None, False, False, False)
def testRunsetBreakGood(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
logger.addExpectedExact('Breaking run set...')
dr.break_existing_runset(cnc)
#self.failUnless(cnc.RSBreakFlag, 'Runset was not broken')
self.assertEquals(0, len(dr.setCompIDs),
'Should not have any components')
self.assertEquals(0, len(dr.shortNameOf),
'Should not have any short names')
self.assertEquals(0, len(dr.daqIDof),
'Should not have any DAQ IDs')
self.assertEquals(0, len(dr.rpcAddrOf),
'Should not have any RPC addresses')
self.assertEquals(0, len(dr.rpcPortOf),
'Should not have any RPC ports')
self.assertEquals(0, len(dr.mbeanPortOf),
'Should not have any MBean ports')
if dr.runSetID is not None: self.fail('Runset ID should be unset')
if dr.lastConfig is not None: self.fail('Last config should be unset')
logger.checkStatus(10)
def testRestartExtraComp(self):
compList = self.__buildCompList(
("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful"))
runConfig = FakeRunConfig("XXXrunCfgXXX")
logger = MockLogger('foo#0')
runset = MyRunSet(MyParent(), runConfig, compList, logger)
extraComp = MockComponent("queen", 10)
longList = compList[:]
longList.append(extraComp)
baseName = "failCluCfg"
clusterCfg = self.__buildClusterConfig(longList, baseName)
logger.addExpectedExact("Cannot remove component %s from RunSet #%d" %
(extraComp.fullName(), runset.id()))
errMsg = None
for c in longList:
if errMsg is None:
errMsg = "Cycling components [" + c.fullName()
else:
errMsg += ", " + c.fullName()
if errMsg is not None:
errMsg += "]"
logger.addExpectedExact(errMsg)
runset.restartComponents(longList, clusterCfg, None, None, None, None,
False, False, False)
def testFillCompDict(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
for i in range(0, len(expComps)):
key = dr.setCompIDs[i]
self.assertEquals(expComps[i][0], key,
'Expected comp#%d to be %s, not %s' %
(i, expComps[i][0], key))
self.assertEquals(expComps[i][1], dr.shortNameOf[key],
'Expected shortName#%d to be %s, not %s' %
(i, expComps[i][1], dr.shortNameOf[key]))
self.assertEquals(expComps[i][2], dr.daqIDof[key],
'Expected daqID#%d to be %d, not %d' %
(i, expComps[i][2], dr.daqIDof[key]))
self.assertEquals(expComps[i][3], dr.rpcAddrOf[key],
'Expected rpcAddr#%d to be %s, not %s' %
(i, expComps[i][3], dr.rpcAddrOf[key]))
self.assertEquals(expComps[i][4], dr.rpcPortOf[key],
'Expected rpcPort#%d to be %d, not %d' %
(i, expComps[i][4], dr.rpcPortOf[key]))
self.assertEquals(expComps[i][5], dr.mbeanPortOf[key],
'Expected mbeanPort#%d to be %d, not %d' %
(i, expComps[i][5], dr.mbeanPortOf[key]))
logger.checkStatus(10)
def testRunSummaryFauxTest(self):
# this is a blatant attempt to increase the code coverage, because the
# rpc_run_summary method is Anvil-centric and I3Live is coming soon
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
dr.runState = "RUNNING"
dr.runStats.runNum = 15
dr.prevRunStats = dr.runStats
dr.rpc_daq_summary_xml()
def testRestartAll(self):
compList = self.__buildCompList(
("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful"))
runConfig = FakeRunConfig("XXXrunCfgXXX")
logger = MockLogger('foo#0')
runset = MyRunSet(MyParent(), runConfig, compList, logger)
clusterCfg = self.__buildClusterConfig(compList, "restartAll")
errMsg = None
for c in compList:
if errMsg is None:
errMsg = "Cycling components [" + c.fullName()
else:
errMsg += ", " + c.fullName()
if errMsg is not None:
errMsg += "]"
logger.addExpectedExact(errMsg)
runset.restartAllComponents(clusterCfg, None, None, None, None, False,
False, False)
def testStartingNoKeys(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
state = { }
self.assertRaisesMsg(LiveException("No stateArgs specified"),
live.starting, state)
def testStartingNoRunNumKey(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
state = { "runConfig": runCfg, }
exc = LiveException("stateArgs does not contain key \"runNumber\"")
self.assertRaisesMsg(exc, live.starting, state)
def testStarting(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
cnc.setRunSet(MockRunSet(runCfg))
state = { "runConfig": runCfg, "runNumber": runNum }
self.failUnless(live.starting(state), "starting failed")
def testStartingNoRunSet(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
state = { "runConfig": runCfg, "runNumber": runNum }
self.assertRaisesMsg(LiveException("Cannot create runset for \"%s\"" %
runCfg), live.starting, state)
def testRPCStopRunStopped(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
logger.addExpectedExact('Warning: run is already stopped.')
dr.runState = 'STOPPED'
dr.rpc_stop_run()
logger.checkStatus(10)
def testSubrun(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
runSet = MockRunSet(runCfg)
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
cnc.setRunSet(runSet)
state = { "runConfig": runCfg, "runNumber": runNum }
self.failUnless(live.starting(state), "starting failed")
self.assertEquals("OK", live.subrun(1, ["domA", "dom2", ]))
def testSetUpAllLoggers(self):
runNum = 5432
dr = MostlyDAQRun()
dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR)
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
nextPort = DAQPort.RUNCOMP_BASE
logger.addExpectedExact('Setting up logging for %d components' %
len(expComps))
for c in expComps:
logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' %
(c[1], c[0], c[3], c[4], dr.ip, nextPort))
nextPort += 1
try:
dr.setUpAllComponentLoggers()
finally:
for k in dr.loggerOf.keys():
if dr.loggerOf[k] is not None:
dr.loggerOf[k].stopServing()
for c in expComps:
path = os.path.join(dr.getLogPath(),
'%s-%d.log' % (c[1], c[2]))
if os.path.exists(path):
os.remove(path)
os.rmdir(dr.getLogPath())
logger.checkStatus(10)
def testBuildMultiMissing(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
compList = []
outputName = "xxx"
comp = MockComponent('fooHub', 0)
comp.addInput(outputName, 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addOutput(outputName)
compList.append(comp)
comp = MockComponent('feeHub', 0)
comp.addInput(outputName, 456)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addOutput(outputName)
compList.append(comp)
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
try:
mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.fail("makeRunset should not succeed")
except ConnectionException, ce:
if str(ce).find(
"Found 2 %s inputs for 2 outputs" % outputName) < 0:
raise ce
def testRunsetConfig(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
logger.addExpectedExact('Configuring run set...')
dr.runset_configure(cnc, 1, 'foo')
self.failUnless(cnc.RSConfigFlag, 'Runset was not configured')
logger.checkStatus(10)
