def __runSubrun(self,
compList,
runNum,
spadeDir="/tmp",
copyDir=None,
expectError=None):
logger = MockLogger('LOG')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runConfig = FakeRunConfig("XXXrunSubXXX")
clusterName = "cluster-foo"
runset = MyRunSet(MyParent(), runConfig, compList, logger)
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
runset.configure()
expState = "ready"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logDir = "/tmp"
#runset.startLogging(logDir, 123, clusterName)
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedRegexp("Could not stop run: .*")
try:
stopErr = runset.stopRun()
except RunSetException, ve:
if not "is not running" in str(ve):
raise ve
stopErr = False
def testBuildMultiInput(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
compList = []
comp = MockComponent('fooHub', 0)
comp.addOutput('conn')
compList.append(comp)
comp = MockComponent('bar', 0)
comp.addInput('conn', 123)
compList.append(comp)
comp = MockComponent('baz', 0)
comp.addInput('conn', 456)
compList.append(comp)
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp("Built runset #\d+: .*")
runset = mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.assertEqual(mgr.numComponents(), 0)
found = mgr.findRunset(runset.id())
self.failIf(found is None, "Couldn't find runset #%d" % runset.id())
mgr.returnRunset(runset, logger)
self.assertEqual(mgr.numComponents(), len(compList))
for c in compList:
mgr.remove(c)
self.assertEqual(mgr.numComponents(), 0)
logger.checkStatus(10)
def testRunsetBreakBad(self):
dr = MostlyDAQRun()
logger = MockLogger('main')
dr.log = logger
expId = 99
expComps = [(3, 'foo', 0, 'localhost', 1234, 5678),
(7, 'bar', 1, 'localhost', 4321, 8765)]
cnc = MockCnCRPC()
cnc.setRunSet(expId, expComps)
cnc.denyBreak = True
dr.runSetID = expId
logger.addExpectedExact('Breaking run set...')
logger.addExpectedRegexp('WARNING: failed to break run set - .*')
dr.break_existing_runset(cnc)
self.failIf(cnc.RSBreakFlag, 'Runset was broken')
self.assertEquals(0, len(dr.setCompIDs),
'Should not have any components')
self.assertEquals(0, len(dr.shortNameOf),
'Should not have any short names')
self.assertEquals(0, len(dr.daqIDof),
'Should not have any DAQ IDs')
self.assertEquals(0, len(dr.rpcAddrOf),
'Should not have any RPC addresses')
self.assertEquals(0, len(dr.rpcPortOf),
'Should not have any RPC ports')
self.assertEquals(0, len(dr.mbeanPortOf),
'Should not have any MBean ports')
if dr.runSetID is not None: self.fail('Runset ID should be unset')
if dr.lastConfig is not None: self.fail('Last config should be unset')
logger.checkStatus(10)
def testStartRun(self):
self.__runConfigDir = tempfile.mkdtemp()
mgr = MyDAQPool()
a = MockComponent('aHub', 0)
a.addOutput('ab')
b = MockComponent('b', 0)
b.addInput('ab', 123)
b.addOutput('bc')
c = MockComponent('c', 0)
c.addInput('bc', 456)
compList = [c, a, b]
self.assertEqual(mgr.numComponents(), 0)
for c in compList:
mgr.add(c)
self.assertEqual(mgr.numComponents(), len(compList))
runConfig = self.__createRunConfigFile(compList)
logger = MockLogger('main')
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp("Built runset #\d+: .*")
runset = mgr.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
self.assertEqual(mgr.numComponents(), 0)
self.assertEqual(runset.size(), len(compList))
self.__checkRunsetState(runset, 'ready')
clusterName = "cluster-foo"
dashLog = runset.getLog("dashLog")
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
r" \d+\S*")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" % clusterName)
self.__checkRunsetState(runset, 'ready')
runNum = 1
moniType = RunOption.MONI_TO_NONE
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, clusterName))
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
spadeDir = "/tmp"
copyDir = None
runset.startRun(runNum, clusterName, moniType, versionInfo, spadeDir,
copyDir)
self.__checkRunsetState(runset, 'running')
numEvts = 0
numSecs = 0
numMoni = 0
numSN = 0
numTcals = 0
dashLog.addExpectedExact("%d physics events collected in %d seconds" %
(numEvts, numSecs))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(runset.stopRun(), "stopRun() encountered error")
self.__checkRunsetState(runset, 'ready')
mgr.returnRunset(runset, logger)
self.assertEqual(runset.id(), None)
self.assertEqual(runset.configured(), False)
self.assertEqual(runset.runNumber(), None)
self.assertEqual(mgr.numComponents(), len(compList))
self.assertEqual(runset.size(), 0)
logger.checkStatus(10)
def __runTests(self, compList, runNum):
logger = MockLogger('foo#0')
num = 1
for c in compList:
c.setOrder(num)
num += 1
runConfig = FakeRunConfig("XXXrunCfgXXX")
expId = RunSet.ID.peekNext()
clusterName = "cluster-foo"
spadeDir = "/tmp"
copyDir = None
runset = MyRunSet(MyParent(), runConfig, compList, logger)
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
expState = "configuring"
i = 0
while True:
cfgWaitStr = None
for c in compList:
if c.getConfigureWait() > i:
if cfgWaitStr is None:
cfgWaitStr = c.fullName()
else:
cfgWaitStr += ', ' + c.fullName()
if cfgWaitStr is None:
break
logger.addExpectedExact("RunSet #%d (%s): Waiting for %s: %s" %
(expId, expState, expState, cfgWaitStr))
i += 1
runset.configure()
expState = "ready"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logDir = "/tmp"
expState = "ready"
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedRegexp("Could not stop run: .*")
self.assertRaises(RunSetException, runset.stopRun)
logger.checkStatus(10)
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
expState = "running"
global CAUGHT_WARNING
if not LIVE_IMPORT and not CAUGHT_WARNING:
CAUGHT_WARNING = True
logger.addExpectedRegexp(r"^Cannot import IceCube Live.*")
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, clusterName))
logger.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
r" \d+\S*")
logger.addExpectedExact("Cluster configuration: %s" % clusterName)
logger.addExpectedExact("Run configuration: %s" % runConfig.basename())
logger.addExpectedExact("Starting run %d..." % runNum)
runset.startRun(runNum, clusterName, RunOption.MONI_TO_NONE,
versionInfo, spadeDir, copyDir, logDir)
self.assertEqual(
str(runset),
'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState))
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failUnless(self.__isCompListRunning(compList, runNum),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
logger.addExpectedExact("0 physics events collected in 0 seconds")
logger.addExpectedExact("0 moni events, 0 SN events, 0 tcals")
logger.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(runset.stopRun(), "stopRun() encountered error")
expState = "ready"
self.assertEqual(
str(runset),
'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState))
if len(compList) > 0:
self.failUnless(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
runset.reset()
expState = "idle"
self.assertEqual(str(runset),
'RunSet #%d (%s)' % (runset.id(), expState))
if len(compList) > 0:
self.failIf(self.__isCompListConfigured(compList),
'Components should be configured')
self.failIf(self.__isCompListRunning(compList),
'Components should not be running')
self.__checkStatus(runset, compList, expState)
logger.checkStatus(10)
def testRunIndirect(self):
self.__copyDir = tempfile.mkdtemp()
self.__runConfigDir = tempfile.mkdtemp()
self.__spadeDir = tempfile.mkdtemp()
comps = [
MockComponent("stringHub", self.HUB_NUMBER,
(MockConn("stringHit", "o"), )),
MockComponent("inIceTrigger",
conn=(MockConn("stringHit",
"i"), MockConn("trigger", "o"))),
MockComponent("globalTrigger",
conn=(MockConn("trigger",
"i"), MockConn("glblTrig", "o"))),
MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )),
MockComponent("extraComp")
]
cluCfg = MockClusterConfig("clusterFoo")
for comp in comps:
cluCfg.addComponent(comp.fullName(), "java", "", "localhost")
self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg,
copyDir=self.__copyDir,
runConfigDir=self.__runConfigDir,
spadeDir=self.__spadeDir)
catchall = self.__cnc.getLogServer()
self.__loadBeanData(comps)
self.__loadRadarDOMMap()
nameList = []
for c in comps:
self.__cnc.add(c)
if c.name() != "stringHub" and c.name() != "extraComp":
nameList.append(str(c))
runCompList = []
for c in comps:
if c.isSource() or c.name() == "extraComp": continue
runCompList.append(c.fullName())
domList = [
MockRunConfigFile.createDOM(self.RADAR_DOM),
]
rcFile = MockRunConfigFile(self.__runConfigDir)
runConfig = rcFile.create(runCompList, domList)
catchall.addExpectedText("Loading run configuration \"%s\"" %
runConfig)
catchall.addExpectedText("Loaded run configuration \"%s\"" % runConfig)
catchall.addExpectedTextRegexp(r"Built runset #\d+: .*")
rsId = self.__cnc.rpc_runset_make(runConfig)
rs = self.__cnc.findRunset(rsId)
self.failIf(rs is None, "Could not find runset #%d" % rsId)
time.sleep(1)
if catchall: catchall.checkStatus(5)
dashLog = MockLogger("dashLog")
rs.setDashLog(dashLog)
liveMoni = SocketReader("liveMoni", DAQPort.I3LIVE, 99)
liveMoni.startServing()
runNum = 345
catchall.addExpectedText("Starting run #%d with \"%s\"" %
(runNum, cluCfg.configName()))
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
" \d+\S+")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" %
cluCfg.configName())
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
self.__cnc.rpc_runset_start_run(rsId, runNum, RunOption.MONI_TO_LIVE)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
numEvts = 5
payTime = 50000000001
firstTime = 1
self.__checkRateTask(comps, rs, liveMoni, dashLog, numEvts, payTime,
firstTime, runNum)
self.__checkMonitorTask(comps, rs, liveMoni)
self.__checkActiveDOMsTask(comps, rs, liveMoni)
self.__checkWatchdogTask(comps, rs, dashLog)
self.__checkRadarTask(comps, rs, liveMoni)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
numMoni = 0
numSN = 0
numTcals = 0
duration = self.__computeDuration(firstTime, payTime)
if duration <= 0:
hzStr = ""
else:
hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration)
dashLog.addExpectedExact(
"%d physics events collected in %d seconds%s" %
(numEvts, duration, hzStr))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.__addRunStopMoni(liveMoni, payTime, numEvts, runNum)
self.__cnc.rpc_runset_stop_run(rsId)
time.sleep(1)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
self.__cnc.rpc_runset_break(rsId)
if catchall: catchall.checkStatus(5)
dashLog.checkStatus(5)
liveMoni.checkStatus(5)
catchall.stopServing()
liveMoni.stopServing()
def __runDirect(self, failReset):
self.__copyDir = tempfile.mkdtemp()
self.__runConfigDir = tempfile.mkdtemp()
self.__spadeDir = tempfile.mkdtemp()
comps = [
MockComponent("stringHub", self.HUB_NUMBER,
(MockConn("stringHit", "o"), )),
MockComponent("inIceTrigger",
conn=(MockConn("stringHit",
"i"), MockConn("trigger", "o"))),
MockComponent("globalTrigger",
conn=(MockConn("trigger",
"i"), MockConn("glblTrig", "o"))),
MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )),
MockComponent("extraComp")
]
cluCfg = MockClusterConfig("clusterFoo")
for comp in comps:
cluCfg.addComponent(comp.fullName(), "java", "", "localhost")
self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg)
self.__loadBeanData(comps)
nameList = []
for c in comps:
self.__cnc.add(c)
if c.name() != "stringHub" and c.name() != "extraComp":
nameList.append(str(c))
domList = [
MockRunConfigFile.createDOM(self.RADAR_DOM),
]
rcFile = MockRunConfigFile(self.__runConfigDir)
runConfig = rcFile.create(nameList, domList)
logger = MockLogger("main")
logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig)
logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig)
logger.addExpectedRegexp(r"Built runset #\d+: .*")
rs = self.__cnc.makeRunset(self.__runConfigDir,
runConfig,
0,
logger,
forceRestart=False,
strict=False)
logger.checkStatus(5)
dashLog = MockLogger("dashLog")
rs.setDashLog(dashLog)
runNum = 321
logger.addExpectedExact("Starting run #%d with \"%s\"" %
(runNum, cluCfg.configName()))
dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" +
" \d+\S+")
dashLog.addExpectedExact("Run configuration: %s" % runConfig)
dashLog.addExpectedExact("Cluster configuration: %s" %
cluCfg.configName())
dashLog.addExpectedExact("Starting run %d..." % runNum)
global ACTIVE_WARNING
if not LIVE_IMPORT and not ACTIVE_WARNING:
ACTIVE_WARNING = True
dashLog.addExpectedExact("Cannot import IceCube Live code, so" +
" per-string active DOM stats wil not" +
" be reported")
versionInfo = {
"filename": "fName",
"revision": "1234",
"date": "date",
"time": "time",
"author": "author",
"release": "rel",
"repo_rev": "1repoRev",
}
rs.startRun(runNum, cluCfg.configName(), RunOption.MONI_TO_NONE,
versionInfo, "/tmp")
logger.checkStatus(5)
dashLog.checkStatus(5)
numEvts = 1000
payTime = 50000000001
firstTime = 1
self.__checkRateTask(comps, rs, None, dashLog, numEvts, payTime,
firstTime, runNum)
numMoni = 0
numSN = 0
numTcals = 0
duration = self.__computeDuration(firstTime, payTime)
if duration <= 0:
hzStr = ""
else:
hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration)
dashLog.addExpectedExact(
"%d physics events collected in %d seconds%s" %
(numEvts, duration, hzStr))
dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" %
(numMoni, numSN, numTcals))
dashLog.addExpectedExact("Run terminated SUCCESSFULLY.")
self.failIf(rs.stopRun(), "stopRun() encountered error")
logger.checkStatus(5)
dashLog.checkStatus(5)
if failReset:
rs.setUnresetComponent(comps[0])
logger.addExpectedExact("Restarting %s (state '%s' after reset)" %
(comps[0], MyRunSet.FAIL_STATE))
logger.addExpectedExact("Cycling components [%s]" % comps[0])
try:
self.__cnc.returnRunset(rs, logger)
if failReset:
self.fail("returnRunset should not have succeeded")
except RunSetException:
if not failReset: raise
logger.checkStatus(5)
dashLog.checkStatus(5)
