def testCheckAllWatchdogMax(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
dr.watchdog = MockWatchdog()
dr.watchdog.inProg = True
dr.watchdog.done = True
DAQRun.unHealthyCount = DAQRun.MAX_UNHEALTHY_COUNT
expCnt = 0
rtnVal = dr.check_all()
self.failIf(rtnVal, 'Expected call to succeed')
self.failUnless(dr.watchdog.threadCleared, 'Should have cleared thread')
self.failIf(dr.watchdog.watchStarted,
'Should not have started watchdog')
self.assertEquals(expCnt, DAQRun.unHealthyCount,
'UnhealthyCount should be %d, not %d' %
(expCnt, DAQRun.unHealthyCount))
logger.checkStatus(10)
def testRunsetBreakGood(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
logger.addExpectedExact('Breaking run set...')
dr.break_existing_runset(cnc)
#self.failUnless(cnc.RSBreakFlag, 'Runset was not broken')
self.assertEquals(0, len(dr.setCompIDs),
'Should not have any components')
self.assertEquals(0, len(dr.shortNameOf),
'Should not have any short names')
self.assertEquals(0, len(dr.daqIDof),
'Should not have any DAQ IDs')
self.assertEquals(0, len(dr.rpcAddrOf),
'Should not have any RPC addresses')
self.assertEquals(0, len(dr.rpcPortOf),
'Should not have any RPC ports')
self.assertEquals(0, len(dr.mbeanPortOf),
'Should not have any MBean ports')
if dr.runSetID is not None: self.fail('Runset ID should be unset')
if dr.lastConfig is not None: self.fail('Last config should be unset')
logger.checkStatus(10)
def testBuildMissingOneOutput(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('aaa')
comp.addInput('xxx', 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('aaa', 456)
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
self.assertRaises(ValueError, mgr.makeRunset, nameList, logger)
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def testVersion(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
self.assertEqual(live.version(),
MockCnC.RELEASE + "_" + MockCnC.REPO_REV)
log.checkStatus(1)
def testCheckAllNone(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
rtnVal = dr.check_all()
self.failUnless(rtnVal, 'Expected call to succeed')
logger.checkStatus(10)
def testBuildMultiInput(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('conn')
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('conn', 123)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addInput('conn', 456)
compList.append(comp)
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp("Built runset #\d+: .*")
runset = mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.assertEqual(mgr.numComponents(), 0)
found = mgr.findRunset(runset.id())
self.failIf(found is None, "Couldn't find runset #%d" % runset.id())
mgr.returnRunset(runset, logger)
self.assertEqual(mgr.numComponents(), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(mgr.numComponents(), 0)
logger.checkStatus(10)
def testRPCStopRunStopped(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
logger.addExpectedExact('Warning: run is already stopped.')
dr.runState = 'STOPPED'
dr.rpc_stop_run()
logger.checkStatus(10)
def testRPCStopRunSuccess(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
dr.runState = 'RUNNING'
dr.rpc_stop_run()
self.assertEquals('STOPPING', dr.runState, 'Should be stopping, not ' +
dr.runState)
logger.checkStatus(10)
def testRunsetConfig(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
logger.addExpectedExact('Configuring run set...')
dr.runset_configure(cnc, 1, 'foo')
self.failUnless(cnc.RSConfigFlag, 'Runset was not configured')
logger.checkStatus(10)
def testCheckAllMoniRate(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
numEvts = 1000
numMoni = 222
numSN = 51
numTCal = 93
dr.moni = MockMoni()
dr.moni.isTime = True
dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts))
dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni))
dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN))
dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal))
dr.rateTimer.trigger()
expId = 99
expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678),
(17, 'secondaryBuilders', 0, 'x', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dt = datetime.datetime.now()
maxRate = 300
for i in range(0, maxRate):
secs = maxRate - i
evts = (maxRate - i) * 2
dr.runStats.physicsRate.add(dt - datetime.timedelta(seconds=secs),
numEvts - evts)
dr.fill_component_dictionaries(cnc)
logger.addExpectedExact(('\t%d physics events (2.00 Hz), %d moni' +
' events, %d SN events, %d tcals') %
(numEvts, numMoni, numSN, numTCal))
rtnVal = dr.check_all()
self.failUnless(rtnVal, 'Expected call to succeed')
logger.checkStatus(10)
def testRPCStopRunBadState(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
badState = 'XXX'
logger.addExpectedExact("Warning: invalid state (%s), won't stop run." %
badState)
dr.runState = badState
dr.rpc_stop_run()
logger.checkStatus(10)
def testBuildMultiMissing(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addInput('xxx', 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addOutput('xxx')
compList.append(comp)
comp = MockComponent('feeHub', 0)
comp.addInput('xxx', 456)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addOutput('xxx')
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
try:
mgr.makeRunset(nameList, logger)
self.fail('Unexpected success')
except ValueError:
pass
except:
self.fail('Unexpected exception')
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def testRunsetStop(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
expRunNum = 100
dr.runStats.runNum = expRunNum
logger.addExpectedExact('Stopping run %d' % expRunNum)
dr.stop_run(cnc)
self.failUnless(cnc.RSStopFlag, 'Runset was not started')
logger.checkStatus(10)
def testBuildMultiInput(self):
mgr = DAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('conn')
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('conn', 123)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addInput('conn', 456)
compList.append(comp)
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
runset = mgr.makeRunset(nameList, logger)
self.assertEqual(len(mgr.pool), 0)
found = mgr.findRunset(runset.id)
self.failIf(found is None, "Couldn't find runset #" + str(runset.id))
mgr.returnRunset(runset)
self.assertEqual(len(mgr.pool), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(len(mgr.pool), 0)
logger.checkStatus(10)
def testSetUpAllLoggers(self):
runNum = 5432
dr = MostlyDAQRun()
dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR)
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
nextPort = DAQPort.RUNCOMP_BASE
logger.addExpectedExact('Setting up logging for %d components' %
len(expComps))
for c in expComps:
logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' %
(c[1], c[0], c[3], c[4], dr.ip, nextPort))
nextPort += 1
try:
dr.setUpAllComponentLoggers()
finally:
for k in dr.loggerOf.keys():
if dr.loggerOf[k] is not None:
dr.loggerOf[k].stopServing()
for c in expComps:
path = os.path.join(dr.getLogPath(),
'%s-%d.log' % (c[1], c[2]))
if os.path.exists(path):
os.remove(path)
os.rmdir(dr.getLogPath())
logger.checkStatus(10)
def testGetEventCounts(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
numEvts = 17
numMoni = 222
numSN = 51
numTCal = 93
dr.moni = MockMoni()
dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts))
dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni))
dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN))
dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal))
expId = 99
expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678),
(17, 'secondaryBuilders', 0, 'x', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
expCnts = (numEvts, numMoni, numSN, numTCal)
cnts = dr.getEventCounts()
self.assertEquals(len(expCnts), len(cnts),
'Expected %d event counts, not %d' %
(len(expCnts), len(cnts)))
for i in range(0, len(expCnts)):
self.assertEquals(expCnts[i], cnts[i],
'Expected event count #%d to be %d, not %d' %
(i, expCnts[i], cnts[i]))
logger.checkStatus(10)
def testRunsetStart(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
expRunNum = 100
expId = 123
dr.runStats.runNum = expRunNum
dr.runSetID = expId
logger.addExpectedExact('Started run %d on run set %d' %
(expRunNum, expId))
dr.start_run(cnc)
self.failUnless(cnc.RSStartFlag, 'Runset was not started')
logger.checkStatus(10)
def testCheckAllMoni(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
numEvts = 17
numMoni = 222
numSN = 51
numTCal = 93
dr.moni = MockMoni()
dr.moni.isTime = True
dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts))
dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni))
dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN))
dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal))
dr.rateTimer.trigger()
expId = 99
expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678),
(17, 'secondaryBuilders', 0, 'x', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
logger.addExpectedExact(('\t%d physics events, %d moni events' +
', %d SN events, %d tcals') %
(numEvts, numMoni, numSN, numTCal))
rtnVal = dr.check_all()
self.failUnless(rtnVal, 'Expected call to succeed')
logger.checkStatus(10)
def testFillCompDict(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
for i in range(0, len(expComps)):
key = dr.setCompIDs[i]
self.assertEquals(expComps[i][0], key,
'Expected comp#%d to be %s, not %s' %
(i, expComps[i][0], key))
self.assertEquals(expComps[i][1], dr.shortNameOf[key],
'Expected shortName#%d to be %s, not %s' %
(i, expComps[i][1], dr.shortNameOf[key]))
self.assertEquals(expComps[i][2], dr.daqIDof[key],
'Expected daqID#%d to be %d, not %d' %
(i, expComps[i][2], dr.daqIDof[key]))
self.assertEquals(expComps[i][3], dr.rpcAddrOf[key],
'Expected rpcAddr#%d to be %s, not %s' %
(i, expComps[i][3], dr.rpcAddrOf[key]))
self.assertEquals(expComps[i][4], dr.rpcPortOf[key],
'Expected rpcPort#%d to be %d, not %d' %
(i, expComps[i][4], dr.rpcPortOf[key]))
self.assertEquals(expComps[i][5], dr.mbeanPortOf[key],
'Expected mbeanPort#%d to be %d, not %d' %
(i, expComps[i][5], dr.mbeanPortOf[key]))
logger.checkStatus(10)
def __runSubrun(self,
compList,
runNum,
spadeDir="/tmp",
copyDir=None,
expectError=None):
logger = MockLogger('LOG')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runConfig = FakeRunConfig("XXXrunSubXXX")
clusterName = "cluster-foo"
runset = MyRunSet(MyParent(), runConfig, compList, logger)
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
runset.configure()
expState = "ready"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logDir = "/tmp"
#runset.startLogging(logDir, 123, clusterName)
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedRegexp("Could not stop run: .*")
try:
stopErr = runset.stopRun()
except RunSetException, ve:
if not "is not running" in str(ve):
raise ve
stopErr = False
def testWaitForRequiredGood(self):
comps = [(0, 'abc', 1, 'xxx', 1, 2),
(1, 'def', 2, 'yyy', 3, 4),
(2, 'ghi', 3, 'zzz', 5, 6)]
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
cnc.setComponents(comps)
expRunNum = 100
dr.runStats.runNum = expRunNum
DAQRun.COMP_TOUT = 0
expId = cnc.nextRunsetId
required = []
for c in comps:
required.append('%s#%d' % (c[1], c[2]))
logger.addExpectedExact(('Starting run %d (waiting for required %d' +
' components to register w/ CnCServer)') %
(dr.runStats.runNum, len(required)))
logger.addExpectedExact('Created Run Set #%d' % expId)
dr.build_run_set(cnc, required)
self.assertEquals(expId, dr.runSetID, 'Expected runset#%d, not #%d' %
(expId, dr.runSetID))
logger.checkStatus(10)
def __runDirect(self, failReset):
self.__copyDir = tempfile.mkdtemp()
self.__runConfigDir = tempfile.mkdtemp()
self.__spadeDir = tempfile.mkdtemp()
comps = [
MockComponent("stringHub", self.HUB_NUMBER,
(MockConn("stringHit", "o"), )),
MockComponent("inIceTrigger",
conn=(MockConn("stringHit",
"i"), MockConn("trigger", "o"))),
MockComponent("globalTrigger",
conn=(MockConn("trigger",
"i"), MockConn("glblTrig", "o"))),
MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )),
MockComponent("extraComp")
]
cluCfg = MockClusterConfig("clusterFoo")
for comp in comps:
cluCfg.addComponent(comp.fullName(), "java", "", "localhost")
self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg)
self.__loadBeanData(comps)
nameList = []
for c in comps:
self.__cnc.add(c)
if c.name() != "stringHub" and c.name() != "extraComp":
nameList.append(str(c))
domList = [
MockRunConfigFile.createDOM(self.RADAR_DOM),
]
rcFile = MockRunConfigFile(self.__runConfigDir)
runConfig = rcFile.create(nameList, domList)
logger = MockLogger("main")
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp(r"Built runset #\d+: .*")
rs = self.__cnc.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
logger.checkStatus(5)
dashLog = MockLogger("dashLog")
rs.setDashLog(dashLog)
runNum = 321
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, cluCfg.configName()))
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
" \d+\S+")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" %
cluCfg.configName())
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
rs.startRun(runNum, cluCfg.configName(), RunOption.MONI_TO_NONE,
versionInfo, "/tmp")
logger.checkStatus(5)
dashLog.checkStatus(5)
numEvts = 1000
payTime = 50000000001
firstTime = 1
self.__checkRateTask(comps, rs, None, dashLog, numEvts, payTime,
firstTime, runNum)
numMoni = 0
numSN = 0
numTcals = 0
duration = self.__computeDuration(firstTime, payTime)
if duration <= 0:
hzStr = ""
else:
hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration)
dashLog.addExpectedExact(
"%d physics events collected in %d seconds%s" %
(numEvts, duration, hzStr))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(rs.stopRun(), "stopRun() encountered error")
logger.checkStatus(5)
dashLog.checkStatus(5)
if failReset:
rs.setUnresetComponent(comps[0])
logger.addExpectedExact("Restarting %s (state '%s' after reset)" %
(comps[0], MyRunSet.FAIL_STATE))
logger.addExpectedExact("Cycling components [%s]" % comps[0])
try:
self.__cnc.returnRunset(rs, logger)
if failReset:
self.fail("returnRunset should not have succeeded")
except RunSetException:
if not failReset: raise
logger.checkStatus(5)
dashLog.checkStatus(5)
def testStartRun(self):
mgr = DAQPool()
a = MockComponent('aHub', 0)
a.addOutput('ab')
b = MockComponent('b', 0)
b.addInput('ab', 123)
b.addOutput('bc')
c = MockComponent('c', 0)
c.addInput('bc', 456)
compList = [c, a, b]
self.assertEqual(len(mgr.pool), 0)
nameList = []
for c in compList:
mgr.add(c)
nameList.append(c.name)
self.assertEqual(len(mgr.pool), len(compList))
logger = MockLogger('main')
runset = mgr.makeRunset(nameList, logger)
self.assertEqual(len(mgr.pool), 0)
self.assertEqual(len(runset.set), len(compList))
runset.configure('abc')
ordered = True
prevName = None
for s in runset.set:
if not prevName:
prevName = s.name
elif prevName > s.name:
ordered = False
self.failIf(ordered, 'Runset sorted before startRun()')
runset.startRun(1)
ordered = True
prevName = None
for s in runset.set:
if not prevName:
prevName = s.name
elif prevName < s.name:
ordered = False
self.failUnless(ordered, 'Runset was not sorted by startRun()')
runset.stopRun()
ordered = True
prevName = None
for s in runset.set:
if not prevName:
prevName = s.name
elif prevName > s.name:
ordered = False
self.failUnless(ordered, 'Runset was not reversed by stopRun()')
mgr.returnRunset(runset)
self.assertEqual(runset.id, None)
self.assertEqual(runset.configured, False)
self.assertEqual(runset.runNumber, None)
self.assertEqual(len(mgr.pool), len(compList))
self.assertEqual(len(runset.set), 0)
logger.checkStatus(10)
def __runTests(self, compList, runNum):
logger = MockLogger('foo#0')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runConfig = FakeRunConfig("XXXrunCfgXXX")
expId = RunSet.ID.peekNext()
clusterName = "cluster-foo"
spadeDir = "/tmp"
copyDir = None
runset = MyRunSet(MyParent(), runConfig, compList, logger)
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
expState = "configuring"
i = 0
while True:
cfgWaitStr = None
for c in compList:
if c.getConfigureWait() > i:
if cfgWaitStr is None:
cfgWaitStr = c.fullName()
else:
cfgWaitStr += ', ' + c.fullName()
if cfgWaitStr is None:
break
logger.addExpectedExact("RunSet #%d (%s): Waiting for %s: %s" %
(expId, expState, expState, cfgWaitStr))
i += 1
runset.configure()
expState = "ready"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logDir = "/tmp"
expState = "ready"
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedRegexp("Could not stop run: .*")
self.assertRaises(RunSetException, runset.stopRun)
logger.checkStatus(10)
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
expState = "running"
global CAUGHT_WARNING
if not LIVE_IMPORT and not CAUGHT_WARNING:
CAUGHT_WARNING = True
logger.addExpectedRegexp(r"^Cannot import IceCube Live.*")
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, clusterName))
logger.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
r" \d+\S*")
logger.addExpectedExact("Cluster configuration: %s" % clusterName)
logger.addExpectedExact("Run configuration: %s" % runConfig.basename())
logger.addExpectedExact("Starting run %d..." % runNum)
runset.startRun(runNum, clusterName, RunOption.MONI_TO_NONE,
versionInfo, spadeDir, copyDir, logDir)
self.assertEqual(
str(runset),
'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState))
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failUnless(self.__isCompListRunning(compList, runNum),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedExact("0 physics events collected in 0 seconds")
logger.addExpectedExact("0 moni events, 0 SN events, 0 tcals")
logger.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(runset.stopRun(), "stopRun() encountered error")
expState = "ready"
self.assertEqual(
str(runset),
'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState))
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
runset.reset()
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
if len(compList) > 0:
self.failIf(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
def buildRunset(self, nodeList):
if LOUD:
print '-- Nodes'
for node in nodeList:
print node.getDescription()
nodeLog = {}
pool = DAQPool()
port = -1
for node in nodeList:
key = '%s#%d' % (node.name, node.num)
nodeLog[key] = MockAppender('Log-%s' % key)
pool.add(MockDAQClient(node.name, node.num, None, port, 0,
node.getConnections(), nodeLog[key],
node.outLinks))
port -= 1
if LOUD:
print '-- Pool has ' + str(len(pool.pool)) + ' comps'
for k in pool.pool.keys():
print ' ' + str(k)
for c in pool.pool[k]:
print ' ' + str(c)
numComps = len(pool.pool)
nameList = []
for node in nodeList:
nameList.append(node.name + '#' + str(node.num))
logger = MockLogger('main')
runset = pool.makeRunset(nameList, logger)
chkId = ConnectionTest.EXP_ID
ConnectionTest.EXP_ID += 1
self.assertEquals(len(pool.pool), 0)
self.assertEquals(len(pool.sets), 1)
self.assertEquals(pool.sets[0], runset)
self.assertEquals(runset.id, chkId)
self.assertEquals(len(runset.set), len(nodeList))
# copy node list
#
tmpList = nodeList[:]
# validate all components in runset
#
for comp in runset.set:
node = None
for t in tmpList:
if comp.name == t.name and comp.num == t.num:
node = t
tmpList.remove(t)
break
self.failIf(not node, 'Could not find component ' + str(comp))
# copy connector list
#
compConn = comp.connectors[:]
# remove all output connectors
#
for typ in node.outLinks:
conn = None
for c in compConn:
if not c.isInput() and c.type == typ:
conn = c
compConn.remove(c)
break
self.failIf(not conn, 'Could not find connector ' + typ +
' for component ' + str(comp))
# remove all input connectors
#
for typ in node.inLinks:
conn = None
for c in compConn:
if c.isInput and c.type == typ:
conn = c
compConn.remove(c)
break
self.failIf(not conn, 'Could not find connector ' + typ +
' for component ' + str(comp))
# whine if any connectors are left
#
self.assertEquals(len(compConn), 0, 'Found extra connectors in ' +
str(compConn))
# whine if any components are left
#
self.assertEquals(len(tmpList), 0, 'Found extra components in ' +
str(tmpList))
if LOUD:
print '-- SET: ' + str(runset)
for key in nodeLog:
nodeLog[key].addExpectedExact('End of log')
pool.returnRunset(runset)
self.assertEquals(len(pool.pool), numComps)
self.assertEquals(len(pool.sets), 0)
logger.checkStatus(10)
for key in nodeLog:
nodeLog[key].checkStatus(10)
def runTests(self, compList, runNum):
logger = MockLogger('foo#0')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runset = RunSet(compList, logger)
self.assertEqual(str(runset), 'RunSet #' + str(runset.id))
self.checkStatus(runset, compList, 'idle')
logList = []
for c in compList:
logList.append([c.name, c.num, 666, 'info'])
runset.configureLogging('localhost', logList)
if len(compList) > 0:
self.failIf(self.isCompListConfigured(compList),
'Components should not be configured')
self.failIf(self.isCompListRunning(compList),
'Components should not be running')
self.assertRaises(ValueError, runset.startRun, 1)
self.assertRaises(ValueError, runset.stopRun)
i = 0
while True:
cfgWaitStr = None
for c in compList:
if c.getConfigureWait() > i:
if cfgWaitStr is None:
cfgWaitStr = c.getName()
else:
cfgWaitStr += ', ' + c.getName()
if cfgWaitStr is None:
break
logger.addExpectedExact('RunSet #%d: Waiting for configuring: %s' %
(runset.id, cfgWaitStr))
i += 1
runset.configure('xxx')
self.assertEqual(str(runset), 'RunSet #' + str(runset.id))
if len(compList) > 0:
self.failUnless(self.isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.isCompListRunning(compList),
'Components should not be running')
self.checkStatus(runset, compList, 'ready')
self.assertRaises(ValueError, runset.stopRun)
runset.startRun(runNum)
self.assertEqual(str(runset),
'RunSet #' + str(runset.id) + ' run#' + str(runNum))
if len(compList) > 0:
self.failUnless(self.isCompListConfigured(compList),
'Components should be configured')
self.failUnless(self.isCompListRunning(compList, runNum),
'Components should not be running')
self.checkStatus(runset, compList, 'running')
runset.stopRun()
self.assertEqual(str(runset), 'RunSet #' + str(runset.id))
if len(compList) > 0:
self.failUnless(self.isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.isCompListRunning(compList),
'Components should not be running')
self.checkStatus(runset, compList, 'ready')
runset.reset()
self.assertEqual(str(runset), 'RunSet #' + str(runset.id))
if len(compList) > 0:
self.failIf(self.isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.isCompListRunning(compList),
'Components should not be running')
self.checkStatus(runset, compList, 'idle')
logger.checkStatus(10)
def testStartRun(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
a = MockComponent('aHub', 0)
a.addOutput('ab')
b = MockComponent('b', 0)
b.addInput('ab', 123)
b.addOutput('bc')
c = MockComponent('c', 0)
c.addInput('bc', 456)
compList = [c, a, b]
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp("Built runset #\d+: .*")
runset = mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.assertEqual(mgr.numComponents(), 0)
self.assertEqual(runset.size(), len(compList))
self.__checkRunsetState(runset, 'ready')
clusterName = "cluster-foo"
dashLog = runset.getLog("dashLog")
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
r" \d+\S*")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" % clusterName)
self.__checkRunsetState(runset, 'ready')
runNum = 1
moniType = RunOption.MONI_TO_NONE
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, clusterName))
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
spadeDir = "/tmp"
copyDir = None
runset.startRun(runNum, clusterName, moniType, versionInfo, spadeDir,
copyDir)
self.__checkRunsetState(runset, 'running')
numEvts = 0
numSecs = 0
numMoni = 0
numSN = 0
numTcals = 0
dashLog.addExpectedExact("%d physics events collected in %d seconds" %
(numEvts, numSecs))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(runset.stopRun(), "stopRun() encountered error")
self.__checkRunsetState(runset, 'ready')
mgr.returnRunset(runset, logger)
self.assertEqual(runset.id(), None)
self.assertEqual(runset.configured(), False)
self.assertEqual(runset.runNumber(), None)
self.assertEqual(mgr.numComponents(), len(compList))
self.assertEqual(runset.size(), 0)
logger.checkStatus(10)
def testSetUpLoggers(self):
runNum = 9753
dr = MostlyDAQRun()
dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR)
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foon', 5, 'localhost', 1234, 5678),
(7, 'barn', 4, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
logger.addExpectedExact('Setting up logging for %d components' %
len(expComps))
nextPort = DAQPort.RUNCOMP_BASE
for i in range(0, len(expComps)):
c = expComps[i]
logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' %
(c[1], c[0], c[3], c[4], dr.ip, nextPort))
nextPort += 1
try:
dr.setup_component_loggers(cnc, 'xxx', expId)
finally:
for k in dr.loggerOf.keys():
if dr.loggerOf[k] is not None:
dr.loggerOf[k].stopServing()
for c in expComps:
path = os.path.join(dr.getLogPath(),
'%s-%d.log' % (c[1], c[2]))
if os.path.exists(path):
os.remove(path)
os.rmdir(dr.getLogPath())
logList = cnc.getRunsetLoggers()
if logList is None:
self.fail('Runset logging was not set')
self.assertEquals(len(expComps), len(logList),
'Expected %d loggers, not %d' %
(len(expComps), len(logList)))
nextPort = DAQPort.RUNCOMP_BASE
for i in range(0, len(expComps)):
c = expComps[i]
l = logList[i]
self.assertEquals(c[1], l[0],
'Expected short name #%d "%s", not "%s"' %
(i, c[1], l[0]))
self.assertEquals(c[2], l[1],
'Expected DAQ ID #%d %d, not %d' %
(i, c[2], l[1]))
self.assertEquals(nextPort, l[2],
'Expected log port #%d %d, not %d' %
(i, nextPort, l[2]))
nextPort += 1
logger.checkStatus(10)
def testRunIndirect(self):
self.__copyDir = tempfile.mkdtemp()
self.__runConfigDir = tempfile.mkdtemp()
self.__spadeDir = tempfile.mkdtemp()
comps = [
MockComponent("stringHub", self.HUB_NUMBER,
(MockConn("stringHit", "o"), )),
MockComponent("inIceTrigger",
conn=(MockConn("stringHit",
"i"), MockConn("trigger", "o"))),
MockComponent("globalTrigger",
conn=(MockConn("trigger",
"i"), MockConn("glblTrig", "o"))),
MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )),
MockComponent("extraComp")
]
cluCfg = MockClusterConfig("clusterFoo")
for comp in comps:
cluCfg.addComponent(comp.fullName(), "java", "", "localhost")
self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg,
copyDir=self.__copyDir,
runConfigDir=self.__runConfigDir,
spadeDir=self.__spadeDir)
catchall = self.__cnc.getLogServer()
self.__loadBeanData(comps)
self.__loadRadarDOMMap()
nameList = []
for c in comps:
self.__cnc.add(c)
if c.name() != "stringHub" and c.name() != "extraComp":
nameList.append(str(c))
runCompList = []
for c in comps:
if c.isSource() or c.name() == "extraComp": continue
runCompList.append(c.fullName())
domList = [
MockRunConfigFile.createDOM(self.RADAR_DOM),
]
rcFile = MockRunConfigFile(self.__runConfigDir)
runConfig = rcFile.create(runCompList, domList)
catchall.addExpectedText("Loading run configuration \"%s\"" %
runConfig)
catchall.addExpectedText("Loaded run configuration \"%s\"" % runConfig)
catchall.addExpectedTextRegexp(r"Built runset #\d+: .*")
rsId = self.__cnc.rpc_runset_make(runConfig)
rs = self.__cnc.findRunset(rsId)
self.failIf(rs is None, "Could not find runset #%d" % rsId)
time.sleep(1)
if catchall: catchall.checkStatus(5)
dashLog = MockLogger("dashLog")
rs.setDashLog(dashLog)
liveMoni = SocketReader("liveMoni", DAQPort.I3LIVE, 99)
liveMoni.startServing()
runNum = 345
catchall.addExpectedText("Starting run #%d with \"%s\"" %
(runNum, cluCfg.configName()))
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
" \d+\S+")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" %
cluCfg.configName())
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
self.__cnc.rpc_runset_start_run(rsId, runNum, RunOption.MONI_TO_LIVE)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
numEvts = 5
payTime = 50000000001
firstTime = 1
self.__checkRateTask(comps, rs, liveMoni, dashLog, numEvts, payTime,
firstTime, runNum)
self.__checkMonitorTask(comps, rs, liveMoni)
self.__checkActiveDOMsTask(comps, rs, liveMoni)
self.__checkWatchdogTask(comps, rs, dashLog)
self.__checkRadarTask(comps, rs, liveMoni)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
numMoni = 0
numSN = 0
numTcals = 0
duration = self.__computeDuration(firstTime, payTime)
if duration <= 0:
hzStr = ""
else:
hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration)
dashLog.addExpectedExact(
"%d physics events collected in %d seconds%s" %
(numEvts, duration, hzStr))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.__addRunStopMoni(liveMoni, payTime, numEvts, runNum)
self.__cnc.rpc_runset_stop_run(rsId)
time.sleep(1)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
self.__cnc.rpc_runset_break(rsId)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
catchall.stopServing()
liveMoni.stopServing()
