def testRestartFailCluCfg(self):
compList = self.__buildCompList(
("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful"))
runConfig = FakeRunConfig("XXXrunCfgXXX")
logger = MockLogger('foo#0')
runset = MyRunSet(MyParent(), runConfig, compList, logger)
baseName = "failCluCfg"
clusterCfg = self.__buildClusterConfig(compList[1:], baseName)
logger.addExpectedExact(
("Cannot restart component %s: Not found" +
" in cluster config \"%s\"") %
(compList[0].fullName(), clusterCfg.configName()))
errMsg = None
for c in compList[1:]:
if errMsg is None:
errMsg = "Cycling components [" + c.fullName()
else:
errMsg += ", " + c.fullName()
if errMsg is not None:
errMsg += "]"
logger.addExpectedExact(errMsg)
runset.restartComponents(compList[:], clusterCfg, None, None, None,
None, False, False, False)
def testRunsetBreakGood(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
logger.addExpectedExact('Breaking run set...')
dr.break_existing_runset(cnc)
#self.failUnless(cnc.RSBreakFlag, 'Runset was not broken')
self.assertEquals(0, len(dr.setCompIDs),
'Should not have any components')
self.assertEquals(0, len(dr.shortNameOf),
'Should not have any short names')
self.assertEquals(0, len(dr.daqIDof),
'Should not have any DAQ IDs')
self.assertEquals(0, len(dr.rpcAddrOf),
'Should not have any RPC addresses')
self.assertEquals(0, len(dr.rpcPortOf),
'Should not have any RPC ports')
self.assertEquals(0, len(dr.mbeanPortOf),
'Should not have any MBean ports')
if dr.runSetID is not None: self.fail('Runset ID should be unset')
if dr.lastConfig is not None: self.fail('Last config should be unset')
logger.checkStatus(10)
def testStoppingNoRunset(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
exc = LiveException("Cannot stop run; no active runset")
self.assertRaisesMsg(exc, live.stopping)
def testStartingNoStateArgs(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
self.assertRaisesMsg(LiveException("No stateArgs specified"),
live.starting, None)
def testRunningNothing(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
exc = LiveException("Cannot check run state; no active runset")
self.assertRaisesMsg(exc, live.running)
def testCheckAllWatchdogMax(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
dr.watchdog = MockWatchdog()
dr.watchdog.inProg = True
dr.watchdog.done = True
DAQRun.unHealthyCount = DAQRun.MAX_UNHEALTHY_COUNT
expCnt = 0
rtnVal = dr.check_all()
self.failIf(rtnVal, 'Expected call to succeed')
self.failUnless(dr.watchdog.threadCleared, 'Should have cleared thread')
self.failIf(dr.watchdog.watchStarted,
'Should not have started watchdog')
self.assertEquals(expCnt, DAQRun.unHealthyCount,
'UnhealthyCount should be %d, not %d' %
(expCnt, DAQRun.unHealthyCount))
logger.checkStatus(10)
def testRestartExtraComp(self):
compList = self.__buildCompList(
("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful"))
runConfig = FakeRunConfig("XXXrunCfgXXX")
logger = MockLogger('foo#0')
runset = MyRunSet(MyParent(), runConfig, compList, logger)
extraComp = MockComponent("queen", 10)
longList = compList[:]
longList.append(extraComp)
baseName = "failCluCfg"
clusterCfg = self.__buildClusterConfig(longList, baseName)
logger.addExpectedExact("Cannot remove component %s from RunSet #%d" %
(extraComp.fullName(), runset.id()))
errMsg = None
for c in longList:
if errMsg is None:
errMsg = "Cycling components [" + c.fullName()
else:
errMsg += ", " + c.fullName()
if errMsg is not None:
errMsg += "]"
logger.addExpectedExact(errMsg)
runset.restartComponents(longList, clusterCfg, None, None, None, None,
False, False, False)
def testWaitForRequiredBad(self):
comps = [(0, 'abc', 1, 'xxx', 1, 2),
(1, 'def', 2, 'yyy', 3, 4),
(2, 'ghi', 3, 'zzz', 5, 6)]
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
cnc.setComponents(comps[1:])
expRunNum = 100
expId = 123
dr.runStats.runNum = expRunNum
dr.runSetID = expId
DAQRun.COMP_TOUT = 0
required = []
for c in comps:
required.append('%s#%d' % (c[1], c[2]))
logger.addExpectedExact(('Starting run %d (waiting for required %d' +
' components to register w/ CnCServer)') %
(dr.runStats.runNum, len(required)))
try:
dr.build_run_set(cnc, required)
self.fail('Unexpected success')
except Exception, e:
self.assertEquals('Still waiting for ' + required[0], str(e),
'Unexpected exception message "%s"' % str(e))
def testBuildMissingOneOutput(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('aaa')
comp.addInput('xxx', 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('aaa', 456)
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
self.assertRaises(ValueError, mgr.makeRunset, nameList, logger)
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def __runSubrun(self,
compList,
runNum,
spadeDir="/tmp",
copyDir=None,
expectError=None):
logger = MockLogger('LOG')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runConfig = FakeRunConfig("XXXrunSubXXX")
clusterName = "cluster-foo"
runset = MyRunSet(MyParent(), runConfig, compList, logger)
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
runset.configure()
expState = "ready"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logDir = "/tmp"
#runset.startLogging(logDir, 123, clusterName)
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedRegexp("Could not stop run: .*")
try:
stopErr = runset.stopRun()
except RunSetException, ve:
if not "is not running" in str(ve):
raise ve
stopErr = False
def testVersion(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
self.assertEqual(live.version(),
MockCnC.RELEASE + "_" + MockCnC.REPO_REV)
log.checkStatus(1)
def testCheckAllNone(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
rtnVal = dr.check_all()
self.failUnless(rtnVal, 'Expected call to succeed')
logger.checkStatus(10)
def testBuildMultiInput(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('conn')
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('conn', 123)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addInput('conn', 456)
compList.append(comp)
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp("Built runset #\d+: .*")
runset = mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.assertEqual(mgr.numComponents(), 0)
found = mgr.findRunset(runset.id())
self.failIf(found is None, "Couldn't find runset #%d" % runset.id())
mgr.returnRunset(runset, logger)
self.assertEqual(mgr.numComponents(), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(mgr.numComponents(), 0)
logger.checkStatus(10)
def testRPCStopRunStopped(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
logger.addExpectedExact('Warning: run is already stopped.')
dr.runState = 'STOPPED'
dr.rpc_stop_run()
logger.checkStatus(10)
def testRPCStopRunSuccess(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
dr.runState = 'RUNNING'
dr.rpc_stop_run()
self.assertEquals('STOPPING', dr.runState, 'Should be stopping, not ' +
dr.runState)
logger.checkStatus(10)
def testRunSummaryFauxTest(self):
# this is a blatant attempt to increase the code coverage, because the
# rpc_run_summary method is Anvil-centric and I3Live is coming soon
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
dr.runState = "RUNNING"
dr.runStats.runNum = 15
dr.prevRunStats = dr.runStats
dr.rpc_daq_summary_xml()
def testBuildMultiMissing(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
compList = []
outputName = "xxx"
comp = MockComponent('fooHub', 0)
comp.addInput(outputName, 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addOutput(outputName)
compList.append(comp)
comp = MockComponent('feeHub', 0)
comp.addInput(outputName, 456)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addOutput(outputName)
compList.append(comp)
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
try:
mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.fail("makeRunset should not succeed")
except ConnectionException, ce:
if str(ce).find(
"Found 2 %s inputs for 2 outputs" % outputName) < 0:
raise ce
def testRunsetConfig(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
logger.addExpectedExact('Configuring run set...')
dr.runset_configure(cnc, 1, 'foo')
self.failUnless(cnc.RSConfigFlag, 'Runset was not configured')
logger.checkStatus(10)
def testCheckAllMoniRate(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
numEvts = 1000
numMoni = 222
numSN = 51
numTCal = 93
dr.moni = MockMoni()
dr.moni.isTime = True
dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts))
dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni))
dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN))
dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal))
dr.rateTimer.trigger()
expId = 99
expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678),
(17, 'secondaryBuilders', 0, 'x', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dt = datetime.datetime.now()
maxRate = 300
for i in range(0, maxRate):
secs = maxRate - i
evts = (maxRate - i) * 2
dr.runStats.physicsRate.add(dt - datetime.timedelta(seconds=secs),
numEvts - evts)
dr.fill_component_dictionaries(cnc)
logger.addExpectedExact(('\t%d physics events (2.00 Hz), %d moni' +
' events, %d SN events, %d tcals') %
(numEvts, numMoni, numSN, numTCal))
rtnVal = dr.check_all()
self.failUnless(rtnVal, 'Expected call to succeed')
logger.checkStatus(10)
def testStartingNoRunNumKey(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
state = { "runConfig": runCfg, }
exc = LiveException("stateArgs does not contain key \"runNumber\"")
self.assertRaisesMsg(exc, live.starting, state)
def testStarting(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
cnc.setRunSet(MockRunSet(runCfg))
state = { "runConfig": runCfg, "runNumber": runNum }
self.failUnless(live.starting(state), "starting failed")
def testStartingNoRunSet(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
state = { "runConfig": runCfg, "runNumber": runNum }
self.assertRaisesMsg(LiveException("Cannot create runset for \"%s\"" %
runCfg), live.starting, state)
def testStartingNoKeys(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
state = { }
self.assertRaisesMsg(LiveException("No stateArgs specified"),
live.starting, state)
def testRPCStopRunBadState(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
badState = 'XXX'
logger.addExpectedExact("Warning: invalid state (%s), won't stop run." %
badState)
dr.runState = badState
dr.rpc_stop_run()
logger.checkStatus(10)
def testBuildMultiMissing(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addInput('xxx', 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addOutput('xxx')
compList.append(comp)
comp = MockComponent('feeHub', 0)
comp.addInput('xxx', 456)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addOutput('xxx')
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
try:
mgr.makeRunset(nameList, logger)
self.fail('Unexpected success')
except ValueError:
pass
except:
self.fail('Unexpected exception')
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def testSubrun(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
runSet = MockRunSet(runCfg)
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
cnc.setRunSet(runSet)
state = { "runConfig": runCfg, "runNumber": runNum }
self.failUnless(live.starting(state), "starting failed")
self.assertEquals("OK", live.subrun(1, ["domA", "dom2", ]))
def testRunsetStop(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
expRunNum = 100
dr.runStats.runNum = expRunNum
logger.addExpectedExact('Stopping run %d' % expRunNum)
dr.stop_run(cnc)
self.failUnless(cnc.RSStopFlag, 'Runset was not started')
logger.checkStatus(10)
def testBuildMultiInput(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('conn')
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('conn', 123)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addInput('conn', 456)
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
runset = mgr.makeRunset(nameList, logger)
self.assertEqual(len(mgr.pool), 0)
found = mgr.findRunset(runset.id)
self.failIf(found is None, "Couldn't find runset #" + str(runset.id))
mgr.returnRunset(runset)
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def testSetUpAllLoggers(self):
runNum = 5432
dr = MostlyDAQRun()
dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR)
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
nextPort = DAQPort.RUNCOMP_BASE
logger.addExpectedExact('Setting up logging for %d components' %
len(expComps))
for c in expComps:
logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' %
(c[1], c[0], c[3], c[4], dr.ip, nextPort))
nextPort += 1
try:
dr.setUpAllComponentLoggers()
finally:
for k in dr.loggerOf.keys():
if dr.loggerOf[k] is not None:
dr.loggerOf[k].stopServing()
for c in expComps:
path = os.path.join(dr.getLogPath(),
'%s-%d.log' % (c[1], c[2]))
if os.path.exists(path):
os.remove(path)
os.rmdir(dr.getLogPath())
logger.checkStatus(10)
def testRunningBadState(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
runSet = MockRunSet(runCfg)
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
cnc.setRunSet(runSet)
state = { "runConfig": runCfg, "runNumber": runNum }
self.failUnless(live.starting(state), "starting failed")
exc = LiveException("%s is not running (state = %s)" %
(runSet, runSet.state()))
self.assertRaisesMsg(exc, live.running)
