def testRunsetBreakGood(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
logger.addExpectedExact('Breaking run set...')
dr.break_existing_runset(cnc)
#self.failUnless(cnc.RSBreakFlag, 'Runset was not broken')
self.assertEquals(0, len(dr.setCompIDs),
'Should not have any components')
self.assertEquals(0, len(dr.shortNameOf),
'Should not have any short names')
self.assertEquals(0, len(dr.daqIDof),
'Should not have any DAQ IDs')
self.assertEquals(0, len(dr.rpcAddrOf),
'Should not have any RPC addresses')
self.assertEquals(0, len(dr.rpcPortOf),
'Should not have any RPC ports')
self.assertEquals(0, len(dr.mbeanPortOf),
'Should not have any MBean ports')
if dr.runSetID is not None: self.fail('Runset ID should be unset')
if dr.lastConfig is not None: self.fail('Last config should be unset')
logger.checkStatus(10)
def testRestartFailCluCfg(self):
compList = self.__buildCompList(
("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful"))
runConfig = FakeRunConfig("XXXrunCfgXXX")
logger = MockLogger('foo#0')
runset = MyRunSet(MyParent(), runConfig, compList, logger)
baseName = "failCluCfg"
clusterCfg = self.__buildClusterConfig(compList[1:], baseName)
logger.addExpectedExact(
("Cannot restart component %s: Not found" +
" in cluster config \"%s\"") %
(compList[0].fullName(), clusterCfg.configName()))
errMsg = None
for c in compList[1:]:
if errMsg is None:
errMsg = "Cycling components [" + c.fullName()
else:
errMsg += ", " + c.fullName()
if errMsg is not None:
errMsg += "]"
logger.addExpectedExact(errMsg)
runset.restartComponents(compList[:], clusterCfg, None, None, None,
None, False, False, False)
def testRestartExtraComp(self):
compList = self.__buildCompList(
("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful"))
runConfig = FakeRunConfig("XXXrunCfgXXX")
logger = MockLogger('foo#0')
runset = MyRunSet(MyParent(), runConfig, compList, logger)
extraComp = MockComponent("queen", 10)
longList = compList[:]
longList.append(extraComp)
baseName = "failCluCfg"
clusterCfg = self.__buildClusterConfig(longList, baseName)
logger.addExpectedExact("Cannot remove component %s from RunSet #%d" %
(extraComp.fullName(), runset.id()))
errMsg = None
for c in longList:
if errMsg is None:
errMsg = "Cycling components [" + c.fullName()
else:
errMsg += ", " + c.fullName()
if errMsg is not None:
errMsg += "]"
logger.addExpectedExact(errMsg)
runset.restartComponents(longList, clusterCfg, None, None, None, None,
False, False, False)
def testWaitForRequiredBad(self):
comps = [(0, 'abc', 1, 'xxx', 1, 2),
(1, 'def', 2, 'yyy', 3, 4),
(2, 'ghi', 3, 'zzz', 5, 6)]
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
cnc.setComponents(comps[1:])
expRunNum = 100
expId = 123
dr.runStats.runNum = expRunNum
dr.runSetID = expId
DAQRun.COMP_TOUT = 0
required = []
for c in comps:
required.append('%s#%d' % (c[1], c[2]))
logger.addExpectedExact(('Starting run %d (waiting for required %d' +
' components to register w/ CnCServer)') %
(dr.runStats.runNum, len(required)))
try:
dr.build_run_set(cnc, required)
self.fail('Unexpected success')
except Exception, e:
self.assertEquals('Still waiting for ' + required[0], str(e),
'Unexpected exception message "%s"' % str(e))
def testBuildMultiInput(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('conn')
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('conn', 123)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addInput('conn', 456)
compList.append(comp)
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp("Built runset #\d+: .*")
runset = mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.assertEqual(mgr.numComponents(), 0)
found = mgr.findRunset(runset.id())
self.failIf(found is None, "Couldn't find runset #%d" % runset.id())
mgr.returnRunset(runset, logger)
self.assertEqual(mgr.numComponents(), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(mgr.numComponents(), 0)
logger.checkStatus(10)
def testRPCStopRunStopped(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
logger.addExpectedExact('Warning: run is already stopped.')
dr.runState = 'STOPPED'
dr.rpc_stop_run()
logger.checkStatus(10)
def testBuildMultiMissing(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
compList = []
outputName = "xxx"
comp = MockComponent('fooHub', 0)
comp.addInput(outputName, 123)
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addOutput(outputName)
compList.append(comp)
comp = MockComponent('feeHub', 0)
comp.addInput(outputName, 456)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addOutput(outputName)
compList.append(comp)
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
try:
mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.fail("makeRunset should not succeed")
except ConnectionException, ce:
if str(ce).find(
"Found 2 %s inputs for 2 outputs" % outputName) < 0:
raise ce
def testRunsetConfig(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
logger.addExpectedExact('Configuring run set...')
dr.runset_configure(cnc, 1, 'foo')
self.failUnless(cnc.RSConfigFlag, 'Runset was not configured')
logger.checkStatus(10)
def testCheckAllMoniRate(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
numEvts = 1000
numMoni = 222
numSN = 51
numTCal = 93
dr.moni = MockMoni()
dr.moni.isTime = True
dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts))
dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni))
dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN))
dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal))
dr.rateTimer.trigger()
expId = 99
expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678),
(17, 'secondaryBuilders', 0, 'x', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dt = datetime.datetime.now()
maxRate = 300
for i in range(0, maxRate):
secs = maxRate - i
evts = (maxRate - i) * 2
dr.runStats.physicsRate.add(dt - datetime.timedelta(seconds=secs),
numEvts - evts)
dr.fill_component_dictionaries(cnc)
logger.addExpectedExact(('\t%d physics events (2.00 Hz), %d moni' +
' events, %d SN events, %d tcals') %
(numEvts, numMoni, numSN, numTCal))
rtnVal = dr.check_all()
self.failUnless(rtnVal, 'Expected call to succeed')
logger.checkStatus(10)
def testRPCStopRunBadState(self):
dr = StubbedDAQRun()
logger = MockLogger('main')
dr.log = logger
badState = 'XXX'
logger.addExpectedExact("Warning: invalid state (%s), won't stop run." %
badState)
dr.runState = badState
dr.rpc_stop_run()
logger.checkStatus(10)
def testRunsetStop(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
expRunNum = 100
dr.runStats.runNum = expRunNum
logger.addExpectedExact('Stopping run %d' % expRunNum)
dr.stop_run(cnc)
self.failUnless(cnc.RSStopFlag, 'Runset was not started')
logger.checkStatus(10)
def testSetUpAllLoggers(self):
runNum = 5432
dr = MostlyDAQRun()
dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR)
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
nextPort = DAQPort.RUNCOMP_BASE
logger.addExpectedExact('Setting up logging for %d components' %
len(expComps))
for c in expComps:
logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' %
(c[1], c[0], c[3], c[4], dr.ip, nextPort))
nextPort += 1
try:
dr.setUpAllComponentLoggers()
finally:
for k in dr.loggerOf.keys():
if dr.loggerOf[k] is not None:
dr.loggerOf[k].stopServing()
for c in expComps:
path = os.path.join(dr.getLogPath(),
'%s-%d.log' % (c[1], c[2]))
if os.path.exists(path):
os.remove(path)
os.rmdir(dr.getLogPath())
logger.checkStatus(10)
def testRecovering(self):
cnc = MockCnC()
log = MockLogger("liveLog")
live = self.__createLive(cnc, log)
runCfg = "foo"
runNum = 13579
runSet = MockRunSet(runCfg)
cnc.setExpectedRunConfig(runCfg)
cnc.setExpectedRunNumber(runNum)
cnc.setRunSet(runSet)
state = { "runConfig": runCfg, "runNumber": runNum }
self.failUnless(live.starting(state), "starting failed")
runSet.setExpectedStopError()
log.addExpectedExact("DAQLive recovered %s" % runSet)
self.failUnless(live.recovering(), "recovering failed")
def testRunsetStart(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
expRunNum = 100
expId = 123
dr.runStats.runNum = expRunNum
dr.runSetID = expId
logger.addExpectedExact('Started run %d on run set %d' %
(expRunNum, expId))
dr.start_run(cnc)
self.failUnless(cnc.RSStartFlag, 'Runset was not started')
logger.checkStatus(10)
def testCheckAllMoni(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
numEvts = 17
numMoni = 222
numSN = 51
numTCal = 93
dr.moni = MockMoni()
dr.moni.isTime = True
dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts))
dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni))
dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN))
dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal))
dr.rateTimer.trigger()
expId = 99
expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678),
(17, 'secondaryBuilders', 0, 'x', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
logger.addExpectedExact(('\t%d physics events, %d moni events' +
', %d SN events, %d tcals') %
(numEvts, numMoni, numSN, numTCal))
rtnVal = dr.check_all()
self.failUnless(rtnVal, 'Expected call to succeed')
logger.checkStatus(10)
def testEmptyRunset(self):
self.__runConfigDir = tempfile.mkdtemp()
self.__cnc = MostlyCnCServer()
nameList = []
rcFile = MockRunConfigFile(self.__runConfigDir)
runConfig = rcFile.create(nameList, [])
logger = MockLogger("main")
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
self.assertRaises(CnCServerException,
self.__cnc.makeRunset,
self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
def testRestartAll(self):
compList = self.__buildCompList(
("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful"))
runConfig = FakeRunConfig("XXXrunCfgXXX")
logger = MockLogger('foo#0')
runset = MyRunSet(MyParent(), runConfig, compList, logger)
clusterCfg = self.__buildClusterConfig(compList, "restartAll")
errMsg = None
for c in compList:
if errMsg is None:
errMsg = "Cycling components [" + c.fullName()
else:
errMsg += ", " + c.fullName()
if errMsg is not None:
errMsg += "]"
logger.addExpectedExact(errMsg)
runset.restartAllComponents(clusterCfg, None, None, None, None, False,
False, False)
def testMissingComponent(self):
self.__runConfigDir = tempfile.mkdtemp()
self.__cnc = MostlyCnCServer()
domList = [
MockRunConfigFile.createDOM(self.RADAR_DOM),
]
rcFile = MockRunConfigFile(self.__runConfigDir)
runConfig = rcFile.create([], domList)
logger = MockLogger("main")
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
self.assertRaises(CnCServerException,
self.__cnc.makeRunset,
self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
def testWaitForRequiredGood(self):
comps = [(0, 'abc', 1, 'xxx', 1, 2),
(1, 'def', 2, 'yyy', 3, 4),
(2, 'ghi', 3, 'zzz', 5, 6)]
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
cnc = MockCnCRPC()
cnc.setComponents(comps)
expRunNum = 100
dr.runStats.runNum = expRunNum
DAQRun.COMP_TOUT = 0
expId = cnc.nextRunsetId
required = []
for c in comps:
required.append('%s#%d' % (c[1], c[2]))
logger.addExpectedExact(('Starting run %d (waiting for required %d' +
' components to register w/ CnCServer)') %
(dr.runStats.runNum, len(required)))
logger.addExpectedExact('Created Run Set #%d' % expId)
dr.build_run_set(cnc, required)
self.assertEquals(expId, dr.runSetID, 'Expected runset#%d, not #%d' %
(expId, dr.runSetID))
logger.checkStatus(10)
def __runDirect(self, failReset):
self.__copyDir = tempfile.mkdtemp()
self.__runConfigDir = tempfile.mkdtemp()
self.__spadeDir = tempfile.mkdtemp()
comps = [
MockComponent("stringHub", self.HUB_NUMBER,
(MockConn("stringHit", "o"), )),
MockComponent("inIceTrigger",
conn=(MockConn("stringHit",
"i"), MockConn("trigger", "o"))),
MockComponent("globalTrigger",
conn=(MockConn("trigger",
"i"), MockConn("glblTrig", "o"))),
MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )),
MockComponent("extraComp")
]
cluCfg = MockClusterConfig("clusterFoo")
for comp in comps:
cluCfg.addComponent(comp.fullName(), "java", "", "localhost")
self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg)
self.__loadBeanData(comps)
nameList = []
for c in comps:
self.__cnc.add(c)
if c.name() != "stringHub" and c.name() != "extraComp":
nameList.append(str(c))
domList = [
MockRunConfigFile.createDOM(self.RADAR_DOM),
]
rcFile = MockRunConfigFile(self.__runConfigDir)
runConfig = rcFile.create(nameList, domList)
logger = MockLogger("main")
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp(r"Built runset #\d+: .*")
rs = self.__cnc.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
logger.checkStatus(5)
dashLog = MockLogger("dashLog")
rs.setDashLog(dashLog)
runNum = 321
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, cluCfg.configName()))
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
" \d+\S+")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" %
cluCfg.configName())
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
rs.startRun(runNum, cluCfg.configName(), RunOption.MONI_TO_NONE,
versionInfo, "/tmp")
logger.checkStatus(5)
dashLog.checkStatus(5)
numEvts = 1000
payTime = 50000000001
firstTime = 1
self.__checkRateTask(comps, rs, None, dashLog, numEvts, payTime,
firstTime, runNum)
numMoni = 0
numSN = 0
numTcals = 0
duration = self.__computeDuration(firstTime, payTime)
if duration <= 0:
hzStr = ""
else:
hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration)
dashLog.addExpectedExact(
"%d physics events collected in %d seconds%s" %
(numEvts, duration, hzStr))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(rs.stopRun(), "stopRun() encountered error")
logger.checkStatus(5)
dashLog.checkStatus(5)
if failReset:
rs.setUnresetComponent(comps[0])
logger.addExpectedExact("Restarting %s (state '%s' after reset)" %
(comps[0], MyRunSet.FAIL_STATE))
logger.addExpectedExact("Cycling components [%s]" % comps[0])
try:
self.__cnc.returnRunset(rs, logger)
if failReset:
self.fail("returnRunset should not have succeeded")
except RunSetException:
if not failReset: raise
logger.checkStatus(5)
dashLog.checkStatus(5)
def runTests(self, compList, runNum):
logger = MockLogger('foo#0')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runset = RunSet(compList, logger)
self.assertEqual(str(runset), 'RunSet #' + str(runset.id))
self.checkStatus(runset, compList, 'idle')
logList = []
for c in compList:
logList.append([c.name, c.num, 666, 'info'])
runset.configureLogging('localhost', logList)
if len(compList) > 0:
self.failIf(self.isCompListConfigured(compList),
'Components should not be configured')
self.failIf(self.isCompListRunning(compList),
'Components should not be running')
self.assertRaises(ValueError, runset.startRun, 1)
self.assertRaises(ValueError, runset.stopRun)
i = 0
while True:
cfgWaitStr = None
for c in compList:
if c.getConfigureWait() > i:
if cfgWaitStr is None:
cfgWaitStr = c.getName()
else:
cfgWaitStr += ', ' + c.getName()
if cfgWaitStr is None:
break
logger.addExpectedExact('RunSet #%d: Waiting for configuring: %s' %
(runset.id, cfgWaitStr))
i += 1
runset.configure('xxx')
self.assertEqual(str(runset), 'RunSet #' + str(runset.id))
if len(compList) > 0:
self.failUnless(self.isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.isCompListRunning(compList),
'Components should not be running')
self.checkStatus(runset, compList, 'ready')
self.assertRaises(ValueError, runset.stopRun)
runset.startRun(runNum)
self.assertEqual(str(runset),
'RunSet #' + str(runset.id) + ' run#' + str(runNum))
if len(compList) > 0:
self.failUnless(self.isCompListConfigured(compList),
'Components should be configured')
self.failUnless(self.isCompListRunning(compList, runNum),
'Components should not be running')
self.checkStatus(runset, compList, 'running')
runset.stopRun()
self.assertEqual(str(runset), 'RunSet #' + str(runset.id))
if len(compList) > 0:
self.failUnless(self.isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.isCompListRunning(compList),
'Components should not be running')
self.checkStatus(runset, compList, 'ready')
runset.reset()
self.assertEqual(str(runset), 'RunSet #' + str(runset.id))
if len(compList) > 0:
self.failIf(self.isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.isCompListRunning(compList),
'Components should not be running')
self.checkStatus(runset, compList, 'idle')
logger.checkStatus(10)
def testStartRun(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
a = MockComponent('aHub', 0)
a.addOutput('ab')
b = MockComponent('b', 0)
b.addInput('ab', 123)
b.addOutput('bc')
c = MockComponent('c', 0)
c.addInput('bc', 456)
compList = [c, a, b]
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp("Built runset #\d+: .*")
runset = mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.assertEqual(mgr.numComponents(), 0)
self.assertEqual(runset.size(), len(compList))
self.__checkRunsetState(runset, 'ready')
clusterName = "cluster-foo"
dashLog = runset.getLog("dashLog")
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
r" \d+\S*")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" % clusterName)
self.__checkRunsetState(runset, 'ready')
runNum = 1
moniType = RunOption.MONI_TO_NONE
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, clusterName))
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
spadeDir = "/tmp"
copyDir = None
runset.startRun(runNum, clusterName, moniType, versionInfo, spadeDir,
copyDir)
self.__checkRunsetState(runset, 'running')
numEvts = 0
numSecs = 0
numMoni = 0
numSN = 0
numTcals = 0
dashLog.addExpectedExact("%d physics events collected in %d seconds" %
(numEvts, numSecs))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(runset.stopRun(), "stopRun() encountered error")
self.__checkRunsetState(runset, 'ready')
mgr.returnRunset(runset, logger)
self.assertEqual(runset.id(), None)
self.assertEqual(runset.configured(), False)
self.assertEqual(runset.runNumber(), None)
self.assertEqual(mgr.numComponents(), len(compList))
self.assertEqual(runset.size(), 0)
logger.checkStatus(10)
def testRunIndirect(self):
self.__copyDir = tempfile.mkdtemp()
self.__runConfigDir = tempfile.mkdtemp()
self.__spadeDir = tempfile.mkdtemp()
comps = [
MockComponent("stringHub", self.HUB_NUMBER,
(MockConn("stringHit", "o"), )),
MockComponent("inIceTrigger",
conn=(MockConn("stringHit",
"i"), MockConn("trigger", "o"))),
MockComponent("globalTrigger",
conn=(MockConn("trigger",
"i"), MockConn("glblTrig", "o"))),
MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )),
MockComponent("extraComp")
]
cluCfg = MockClusterConfig("clusterFoo")
for comp in comps:
cluCfg.addComponent(comp.fullName(), "java", "", "localhost")
self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg,
copyDir=self.__copyDir,
runConfigDir=self.__runConfigDir,
spadeDir=self.__spadeDir)
catchall = self.__cnc.getLogServer()
self.__loadBeanData(comps)
self.__loadRadarDOMMap()
nameList = []
for c in comps:
self.__cnc.add(c)
if c.name() != "stringHub" and c.name() != "extraComp":
nameList.append(str(c))
runCompList = []
for c in comps:
if c.isSource() or c.name() == "extraComp": continue
runCompList.append(c.fullName())
domList = [
MockRunConfigFile.createDOM(self.RADAR_DOM),
]
rcFile = MockRunConfigFile(self.__runConfigDir)
runConfig = rcFile.create(runCompList, domList)
catchall.addExpectedText("Loading run configuration \"%s\"" %
runConfig)
catchall.addExpectedText("Loaded run configuration \"%s\"" % runConfig)
catchall.addExpectedTextRegexp(r"Built runset #\d+: .*")
rsId = self.__cnc.rpc_runset_make(runConfig)
rs = self.__cnc.findRunset(rsId)
self.failIf(rs is None, "Could not find runset #%d" % rsId)
time.sleep(1)
if catchall: catchall.checkStatus(5)
dashLog = MockLogger("dashLog")
rs.setDashLog(dashLog)
liveMoni = SocketReader("liveMoni", DAQPort.I3LIVE, 99)
liveMoni.startServing()
runNum = 345
catchall.addExpectedText("Starting run #%d with \"%s\"" %
(runNum, cluCfg.configName()))
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
" \d+\S+")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" %
cluCfg.configName())
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
self.__cnc.rpc_runset_start_run(rsId, runNum, RunOption.MONI_TO_LIVE)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
numEvts = 5
payTime = 50000000001
firstTime = 1
self.__checkRateTask(comps, rs, liveMoni, dashLog, numEvts, payTime,
firstTime, runNum)
self.__checkMonitorTask(comps, rs, liveMoni)
self.__checkActiveDOMsTask(comps, rs, liveMoni)
self.__checkWatchdogTask(comps, rs, dashLog)
self.__checkRadarTask(comps, rs, liveMoni)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
numMoni = 0
numSN = 0
numTcals = 0
duration = self.__computeDuration(firstTime, payTime)
if duration <= 0:
hzStr = ""
else:
hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration)
dashLog.addExpectedExact(
"%d physics events collected in %d seconds%s" %
(numEvts, duration, hzStr))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.__addRunStopMoni(liveMoni, payTime, numEvts, runNum)
self.__cnc.rpc_runset_stop_run(rsId)
time.sleep(1)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
self.__cnc.rpc_runset_break(rsId)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
catchall.stopServing()
liveMoni.stopServing()
def __runTests(self, compList, runNum):
logger = MockLogger('foo#0')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runConfig = FakeRunConfig("XXXrunCfgXXX")
expId = RunSet.ID.peekNext()
clusterName = "cluster-foo"
spadeDir = "/tmp"
copyDir = None
runset = MyRunSet(MyParent(), runConfig, compList, logger)
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
expState = "configuring"
i = 0
while True:
cfgWaitStr = None
for c in compList:
if c.getConfigureWait() > i:
if cfgWaitStr is None:
cfgWaitStr = c.fullName()
else:
cfgWaitStr += ', ' + c.fullName()
if cfgWaitStr is None:
break
logger.addExpectedExact("RunSet #%d (%s): Waiting for %s: %s" %
(expId, expState, expState, cfgWaitStr))
i += 1
runset.configure()
expState = "ready"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logDir = "/tmp"
expState = "ready"
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedRegexp("Could not stop run: .*")
self.assertRaises(RunSetException, runset.stopRun)
logger.checkStatus(10)
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
expState = "running"
global CAUGHT_WARNING
if not LIVE_IMPORT and not CAUGHT_WARNING:
CAUGHT_WARNING = True
logger.addExpectedRegexp(r"^Cannot import IceCube Live.*")
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, clusterName))
logger.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
r" \d+\S*")
logger.addExpectedExact("Cluster configuration: %s" % clusterName)
logger.addExpectedExact("Run configuration: %s" % runConfig.basename())
logger.addExpectedExact("Starting run %d..." % runNum)
runset.startRun(runNum, clusterName, RunOption.MONI_TO_NONE,
versionInfo, spadeDir, copyDir, logDir)
self.assertEqual(
str(runset),
'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState))
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failUnless(self.__isCompListRunning(compList, runNum),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedExact("0 physics events collected in 0 seconds")
logger.addExpectedExact("0 moni events, 0 SN events, 0 tcals")
logger.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(runset.stopRun(), "stopRun() encountered error")
expState = "ready"
self.assertEqual(
str(runset),
'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState))
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
runset.reset()
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
if len(compList) > 0:
self.failIf(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
def testSetUpLoggers(self):
runNum = 9753
dr = MostlyDAQRun()
dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR)
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foon', 5, 'localhost', 1234, 5678),
(7, 'barn', 4, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
dr.runSetID = expId
dr.fill_component_dictionaries(cnc)
logger.addExpectedExact('Setting up logging for %d components' %
len(expComps))
nextPort = DAQPort.RUNCOMP_BASE
for i in range(0, len(expComps)):
c = expComps[i]
logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' %
(c[1], c[0], c[3], c[4], dr.ip, nextPort))
nextPort += 1
try:
dr.setup_component_loggers(cnc, 'xxx', expId)
finally:
for k in dr.loggerOf.keys():
if dr.loggerOf[k] is not None:
dr.loggerOf[k].stopServing()
for c in expComps:
path = os.path.join(dr.getLogPath(),
'%s-%d.log' % (c[1], c[2]))
if os.path.exists(path):
os.remove(path)
os.rmdir(dr.getLogPath())
logList = cnc.getRunsetLoggers()
if logList is None:
self.fail('Runset logging was not set')
self.assertEquals(len(expComps), len(logList),
'Expected %d loggers, not %d' %
(len(expComps), len(logList)))
nextPort = DAQPort.RUNCOMP_BASE
for i in range(0, len(expComps)):
c = expComps[i]
l = logList[i]
self.assertEquals(c[1], l[0],
'Expected short name #%d "%s", not "%s"' %
(i, c[1], l[0]))
self.assertEquals(c[2], l[1],
'Expected DAQ ID #%d %d, not %d' %
(i, c[2], l[1]))
self.assertEquals(nextPort, l[2],
'Expected log port #%d %d, not %d' %
(i, nextPort, l[2]))
nextPort += 1
logger.checkStatus(10)
