def testCheckAllWatchdogMax(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger dr.watchdog = MockWatchdog() dr.watchdog.inProg = True dr.watchdog.done = True DAQRun.unHealthyCount = DAQRun.MAX_UNHEALTHY_COUNT expCnt = 0 rtnVal = dr.check_all() self.failIf(rtnVal, 'Expected call to succeed') self.failUnless(dr.watchdog.threadCleared, 'Should have cleared thread') self.failIf(dr.watchdog.watchStarted, 'Should not have started watchdog') self.assertEquals(expCnt, DAQRun.unHealthyCount, 'UnhealthyCount should be %d, not %d' % (expCnt, DAQRun.unHealthyCount)) logger.checkStatus(10)
def testRunsetBreakGood(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger expId = 99 expComps = [(3, 'foo', 0, 'localhost', 1234, 5678), (7, 'bar', 1, 'localhost', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId logger.addExpectedExact('Breaking run set...') dr.break_existing_runset(cnc) #self.failUnless(cnc.RSBreakFlag, 'Runset was not broken') self.assertEquals(0, len(dr.setCompIDs), 'Should not have any components') self.assertEquals(0, len(dr.shortNameOf), 'Should not have any short names') self.assertEquals(0, len(dr.daqIDof), 'Should not have any DAQ IDs') self.assertEquals(0, len(dr.rpcAddrOf), 'Should not have any RPC addresses') self.assertEquals(0, len(dr.rpcPortOf), 'Should not have any RPC ports') self.assertEquals(0, len(dr.mbeanPortOf), 'Should not have any MBean ports') if dr.runSetID is not None: self.fail('Runset ID should be unset') if dr.lastConfig is not None: self.fail('Last config should be unset') logger.checkStatus(10)
def testBuildMissingOneOutput(self): mgr = DAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addOutput('aaa') comp.addInput('xxx', 123) compList.append(comp) comp = MockComponent('bar', 0) comp.addInput('aaa', 456) compList.append(comp) self.assertEqual(len(mgr.pool), 0) nameList = [] for c in compList: mgr.add(c) nameList.append(c.name) self.assertEqual(len(mgr.pool), len(compList)) logger = MockLogger('main') self.assertRaises(ValueError, mgr.makeRunset, nameList, logger) self.assertEqual(len(mgr.pool), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(len(mgr.pool), 0) logger.checkStatus(10)
def testVersion(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) self.assertEqual(live.version(), MockCnC.RELEASE + "_" + MockCnC.REPO_REV) log.checkStatus(1)
def testCheckAllNone(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger rtnVal = dr.check_all() self.failUnless(rtnVal, 'Expected call to succeed') logger.checkStatus(10)
def testBuildMultiInput(self): self.__runConfigDir = tempfile.mkdtemp() mgr = MyDAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addOutput('conn') compList.append(comp) comp = MockComponent('bar', 0) comp.addInput('conn', 123) compList.append(comp) comp = MockComponent('baz', 0) comp.addInput('conn', 456) compList.append(comp) self.assertEqual(mgr.numComponents(), 0) for c in compList: mgr.add(c) self.assertEqual(mgr.numComponents(), len(compList)) runConfig = self.__createRunConfigFile(compList) logger = MockLogger('main') logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig) logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig) logger.addExpectedRegexp("Built runset #\d+: .*") runset = mgr.makeRunset(self.__runConfigDir, runConfig, 0, logger, forceRestart=False, strict=False) self.assertEqual(mgr.numComponents(), 0) found = mgr.findRunset(runset.id()) self.failIf(found is None, "Couldn't find runset #%d" % runset.id()) mgr.returnRunset(runset, logger) self.assertEqual(mgr.numComponents(), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(mgr.numComponents(), 0) logger.checkStatus(10)
def testRPCStopRunStopped(self): dr = StubbedDAQRun() logger = MockLogger('main') dr.log = logger logger.addExpectedExact('Warning: run is already stopped.') dr.runState = 'STOPPED' dr.rpc_stop_run() logger.checkStatus(10)
def testRPCStopRunSuccess(self): dr = StubbedDAQRun() logger = MockLogger('main') dr.log = logger dr.runState = 'RUNNING' dr.rpc_stop_run() self.assertEquals('STOPPING', dr.runState, 'Should be stopping, not ' + dr.runState) logger.checkStatus(10)
def testRunsetConfig(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger cnc = MockCnCRPC() logger.addExpectedExact('Configuring run set...') dr.runset_configure(cnc, 1, 'foo') self.failUnless(cnc.RSConfigFlag, 'Runset was not configured') logger.checkStatus(10)
def testCheckAllMoniRate(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger numEvts = 1000 numMoni = 222 numSN = 51 numTCal = 93 dr.moni = MockMoni() dr.moni.isTime = True dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts)) dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni)) dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN)) dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal)) dr.rateTimer.trigger() expId = 99 expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678), (17, 'secondaryBuilders', 0, 'x', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dt = datetime.datetime.now() maxRate = 300 for i in range(0, maxRate): secs = maxRate - i evts = (maxRate - i) * 2 dr.runStats.physicsRate.add(dt - datetime.timedelta(seconds=secs), numEvts - evts) dr.fill_component_dictionaries(cnc) logger.addExpectedExact(('\t%d physics events (2.00 Hz), %d moni' + ' events, %d SN events, %d tcals') % (numEvts, numMoni, numSN, numTCal)) rtnVal = dr.check_all() self.failUnless(rtnVal, 'Expected call to succeed') logger.checkStatus(10)
def testRPCStopRunBadState(self): dr = StubbedDAQRun() logger = MockLogger('main') dr.log = logger badState = 'XXX' logger.addExpectedExact("Warning: invalid state (%s), won't stop run." % badState) dr.runState = badState dr.rpc_stop_run() logger.checkStatus(10)
def testBuildMultiMissing(self): mgr = DAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addInput('xxx', 123) compList.append(comp) comp = MockComponent('bar', 0) comp.addOutput('xxx') compList.append(comp) comp = MockComponent('feeHub', 0) comp.addInput('xxx', 456) compList.append(comp) comp = MockComponent('baz', 0) comp.addOutput('xxx') compList.append(comp) self.assertEqual(len(mgr.pool), 0) nameList = [] for c in compList: mgr.add(c) nameList.append(c.name) self.assertEqual(len(mgr.pool), len(compList)) logger = MockLogger('main') try: mgr.makeRunset(nameList, logger) self.fail('Unexpected success') except ValueError: pass except: self.fail('Unexpected exception') self.assertEqual(len(mgr.pool), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(len(mgr.pool), 0) logger.checkStatus(10)
def testRunsetStop(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger cnc = MockCnCRPC() expRunNum = 100 dr.runStats.runNum = expRunNum logger.addExpectedExact('Stopping run %d' % expRunNum) dr.stop_run(cnc) self.failUnless(cnc.RSStopFlag, 'Runset was not started') logger.checkStatus(10)
def testBuildMultiInput(self): mgr = DAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addOutput('conn') compList.append(comp) comp = MockComponent('bar', 0) comp.addInput('conn', 123) compList.append(comp) comp = MockComponent('baz', 0) comp.addInput('conn', 456) compList.append(comp) self.assertEqual(len(mgr.pool), 0) nameList = [] for c in compList: mgr.add(c) nameList.append(c.name) self.assertEqual(len(mgr.pool), len(compList)) logger = MockLogger('main') runset = mgr.makeRunset(nameList, logger) self.assertEqual(len(mgr.pool), 0) found = mgr.findRunset(runset.id) self.failIf(found is None, "Couldn't find runset #" + str(runset.id)) mgr.returnRunset(runset) self.assertEqual(len(mgr.pool), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(len(mgr.pool), 0) logger.checkStatus(10)
def testSetUpAllLoggers(self): runNum = 5432 dr = MostlyDAQRun() dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR) logger = MockLogger('main') dr.log = logger expId = 99 expComps = [(3, 'foo', 0, 'localhost', 1234, 5678), (7, 'bar', 1, 'localhost', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dr.fill_component_dictionaries(cnc) nextPort = DAQPort.RUNCOMP_BASE logger.addExpectedExact('Setting up logging for %d components' % len(expComps)) for c in expComps: logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' % (c[1], c[0], c[3], c[4], dr.ip, nextPort)) nextPort += 1 try: dr.setUpAllComponentLoggers() finally: for k in dr.loggerOf.keys(): if dr.loggerOf[k] is not None: dr.loggerOf[k].stopServing() for c in expComps: path = os.path.join(dr.getLogPath(), '%s-%d.log' % (c[1], c[2])) if os.path.exists(path): os.remove(path) os.rmdir(dr.getLogPath()) logger.checkStatus(10)
def testGetEventCounts(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger numEvts = 17 numMoni = 222 numSN = 51 numTCal = 93 dr.moni = MockMoni() dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts)) dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni)) dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN)) dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal)) expId = 99 expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678), (17, 'secondaryBuilders', 0, 'x', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dr.fill_component_dictionaries(cnc) expCnts = (numEvts, numMoni, numSN, numTCal) cnts = dr.getEventCounts() self.assertEquals(len(expCnts), len(cnts), 'Expected %d event counts, not %d' % (len(expCnts), len(cnts))) for i in range(0, len(expCnts)): self.assertEquals(expCnts[i], cnts[i], 'Expected event count #%d to be %d, not %d' % (i, expCnts[i], cnts[i])) logger.checkStatus(10)
def testRunsetStart(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger cnc = MockCnCRPC() expRunNum = 100 expId = 123 dr.runStats.runNum = expRunNum dr.runSetID = expId logger.addExpectedExact('Started run %d on run set %d' % (expRunNum, expId)) dr.start_run(cnc) self.failUnless(cnc.RSStartFlag, 'Runset was not started') logger.checkStatus(10)
def testCheckAllMoni(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger numEvts = 17 numMoni = 222 numSN = 51 numTCal = 93 dr.moni = MockMoni() dr.moni.isTime = True dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts)) dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni)) dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN)) dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal)) dr.rateTimer.trigger() expId = 99 expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678), (17, 'secondaryBuilders', 0, 'x', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dr.fill_component_dictionaries(cnc) logger.addExpectedExact(('\t%d physics events, %d moni events' + ', %d SN events, %d tcals') % (numEvts, numMoni, numSN, numTCal)) rtnVal = dr.check_all() self.failUnless(rtnVal, 'Expected call to succeed') logger.checkStatus(10)
def testFillCompDict(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger expId = 99 expComps = [(3, 'foo', 0, 'localhost', 1234, 5678), (7, 'bar', 1, 'localhost', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dr.fill_component_dictionaries(cnc) for i in range(0, len(expComps)): key = dr.setCompIDs[i] self.assertEquals(expComps[i][0], key, 'Expected comp#%d to be %s, not %s' % (i, expComps[i][0], key)) self.assertEquals(expComps[i][1], dr.shortNameOf[key], 'Expected shortName#%d to be %s, not %s' % (i, expComps[i][1], dr.shortNameOf[key])) self.assertEquals(expComps[i][2], dr.daqIDof[key], 'Expected daqID#%d to be %d, not %d' % (i, expComps[i][2], dr.daqIDof[key])) self.assertEquals(expComps[i][3], dr.rpcAddrOf[key], 'Expected rpcAddr#%d to be %s, not %s' % (i, expComps[i][3], dr.rpcAddrOf[key])) self.assertEquals(expComps[i][4], dr.rpcPortOf[key], 'Expected rpcPort#%d to be %d, not %d' % (i, expComps[i][4], dr.rpcPortOf[key])) self.assertEquals(expComps[i][5], dr.mbeanPortOf[key], 'Expected mbeanPort#%d to be %d, not %d' % (i, expComps[i][5], dr.mbeanPortOf[key])) logger.checkStatus(10)
def __runSubrun(self, compList, runNum, spadeDir="/tmp", copyDir=None, expectError=None): logger = MockLogger('LOG') num = 1 for c in compList: c.setOrder(num) num += 1 runConfig = FakeRunConfig("XXXrunSubXXX") clusterName = "cluster-foo" runset = MyRunSet(MyParent(), runConfig, compList, logger) expState = "idle" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) runset.configure() expState = "ready" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logDir = "/tmp" #runset.startLogging(logDir, 123, clusterName) if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logger.addExpectedRegexp("Could not stop run: .*") try: stopErr = runset.stopRun() except RunSetException, ve: if not "is not running" in str(ve): raise ve stopErr = False
def testWaitForRequiredGood(self): comps = [(0, 'abc', 1, 'xxx', 1, 2), (1, 'def', 2, 'yyy', 3, 4), (2, 'ghi', 3, 'zzz', 5, 6)] dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger cnc = MockCnCRPC() cnc.setComponents(comps) expRunNum = 100 dr.runStats.runNum = expRunNum DAQRun.COMP_TOUT = 0 expId = cnc.nextRunsetId required = [] for c in comps: required.append('%s#%d' % (c[1], c[2])) logger.addExpectedExact(('Starting run %d (waiting for required %d' + ' components to register w/ CnCServer)') % (dr.runStats.runNum, len(required))) logger.addExpectedExact('Created Run Set #%d' % expId) dr.build_run_set(cnc, required) self.assertEquals(expId, dr.runSetID, 'Expected runset#%d, not #%d' % (expId, dr.runSetID)) logger.checkStatus(10)
def __runDirect(self, failReset): self.__copyDir = tempfile.mkdtemp() self.__runConfigDir = tempfile.mkdtemp() self.__spadeDir = tempfile.mkdtemp() comps = [ MockComponent("stringHub", self.HUB_NUMBER, (MockConn("stringHit", "o"), )), MockComponent("inIceTrigger", conn=(MockConn("stringHit", "i"), MockConn("trigger", "o"))), MockComponent("globalTrigger", conn=(MockConn("trigger", "i"), MockConn("glblTrig", "o"))), MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )), MockComponent("extraComp") ] cluCfg = MockClusterConfig("clusterFoo") for comp in comps: cluCfg.addComponent(comp.fullName(), "java", "", "localhost") self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg) self.__loadBeanData(comps) nameList = [] for c in comps: self.__cnc.add(c) if c.name() != "stringHub" and c.name() != "extraComp": nameList.append(str(c)) domList = [ MockRunConfigFile.createDOM(self.RADAR_DOM), ] rcFile = MockRunConfigFile(self.__runConfigDir) runConfig = rcFile.create(nameList, domList) logger = MockLogger("main") logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig) logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig) logger.addExpectedRegexp(r"Built runset #\d+: .*") rs = self.__cnc.makeRunset(self.__runConfigDir, runConfig, 0, logger, forceRestart=False, strict=False) logger.checkStatus(5) dashLog = MockLogger("dashLog") rs.setDashLog(dashLog) runNum = 321 logger.addExpectedExact("Starting run #%d with \"%s\"" % (runNum, cluCfg.configName())) dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" + " \d+\S+") dashLog.addExpectedExact("Run configuration: %s" % runConfig) dashLog.addExpectedExact("Cluster configuration: %s" % cluCfg.configName()) dashLog.addExpectedExact("Starting run %d..." % runNum) global ACTIVE_WARNING if not LIVE_IMPORT and not ACTIVE_WARNING: ACTIVE_WARNING = True dashLog.addExpectedExact("Cannot import IceCube Live code, so" + " per-string active DOM stats wil not" + " be reported") versionInfo = { "filename": "fName", "revision": "1234", "date": "date", "time": "time", "author": "author", "release": "rel", "repo_rev": "1repoRev", } rs.startRun(runNum, cluCfg.configName(), RunOption.MONI_TO_NONE, versionInfo, "/tmp") logger.checkStatus(5) dashLog.checkStatus(5) numEvts = 1000 payTime = 50000000001 firstTime = 1 self.__checkRateTask(comps, rs, None, dashLog, numEvts, payTime, firstTime, runNum) numMoni = 0 numSN = 0 numTcals = 0 duration = self.__computeDuration(firstTime, payTime) if duration <= 0: hzStr = "" else: hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration) dashLog.addExpectedExact( "%d physics events collected in %d seconds%s" % (numEvts, duration, hzStr)) dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" % (numMoni, numSN, numTcals)) dashLog.addExpectedExact("Run terminated SUCCESSFULLY.") self.failIf(rs.stopRun(), "stopRun() encountered error") logger.checkStatus(5) dashLog.checkStatus(5) if failReset: rs.setUnresetComponent(comps[0]) logger.addExpectedExact("Restarting %s (state '%s' after reset)" % (comps[0], MyRunSet.FAIL_STATE)) logger.addExpectedExact("Cycling components [%s]" % comps[0]) try: self.__cnc.returnRunset(rs, logger) if failReset: self.fail("returnRunset should not have succeeded") except RunSetException: if not failReset: raise logger.checkStatus(5) dashLog.checkStatus(5)
def testStartRun(self): mgr = DAQPool() a = MockComponent('aHub', 0) a.addOutput('ab') b = MockComponent('b', 0) b.addInput('ab', 123) b.addOutput('bc') c = MockComponent('c', 0) c.addInput('bc', 456) compList = [c, a, b] self.assertEqual(len(mgr.pool), 0) nameList = [] for c in compList: mgr.add(c) nameList.append(c.name) self.assertEqual(len(mgr.pool), len(compList)) logger = MockLogger('main') runset = mgr.makeRunset(nameList, logger) self.assertEqual(len(mgr.pool), 0) self.assertEqual(len(runset.set), len(compList)) runset.configure('abc') ordered = True prevName = None for s in runset.set: if not prevName: prevName = s.name elif prevName > s.name: ordered = False self.failIf(ordered, 'Runset sorted before startRun()') runset.startRun(1) ordered = True prevName = None for s in runset.set: if not prevName: prevName = s.name elif prevName < s.name: ordered = False self.failUnless(ordered, 'Runset was not sorted by startRun()') runset.stopRun() ordered = True prevName = None for s in runset.set: if not prevName: prevName = s.name elif prevName > s.name: ordered = False self.failUnless(ordered, 'Runset was not reversed by stopRun()') mgr.returnRunset(runset) self.assertEqual(runset.id, None) self.assertEqual(runset.configured, False) self.assertEqual(runset.runNumber, None) self.assertEqual(len(mgr.pool), len(compList)) self.assertEqual(len(runset.set), 0) logger.checkStatus(10)
def __runTests(self, compList, runNum): logger = MockLogger('foo#0') num = 1 for c in compList: c.setOrder(num) num += 1 runConfig = FakeRunConfig("XXXrunCfgXXX") expId = RunSet.ID.peekNext() clusterName = "cluster-foo" spadeDir = "/tmp" copyDir = None runset = MyRunSet(MyParent(), runConfig, compList, logger) expState = "idle" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) expState = "configuring" i = 0 while True: cfgWaitStr = None for c in compList: if c.getConfigureWait() > i: if cfgWaitStr is None: cfgWaitStr = c.fullName() else: cfgWaitStr += ', ' + c.fullName() if cfgWaitStr is None: break logger.addExpectedExact("RunSet #%d (%s): Waiting for %s: %s" % (expId, expState, expState, cfgWaitStr)) i += 1 runset.configure() expState = "ready" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logDir = "/tmp" expState = "ready" if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logger.addExpectedRegexp("Could not stop run: .*") self.assertRaises(RunSetException, runset.stopRun) logger.checkStatus(10) versionInfo = { "filename": "fName", "revision": "1234", "date": "date", "time": "time", "author": "author", "release": "rel", "repo_rev": "1repoRev", } expState = "running" global CAUGHT_WARNING if not LIVE_IMPORT and not CAUGHT_WARNING: CAUGHT_WARNING = True logger.addExpectedRegexp(r"^Cannot import IceCube Live.*") logger.addExpectedExact("Starting run #%d with \"%s\"" % (runNum, clusterName)) logger.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" + r" \d+\S*") logger.addExpectedExact("Cluster configuration: %s" % clusterName) logger.addExpectedExact("Run configuration: %s" % runConfig.basename()) logger.addExpectedExact("Starting run %d..." % runNum) runset.startRun(runNum, clusterName, RunOption.MONI_TO_NONE, versionInfo, spadeDir, copyDir, logDir) self.assertEqual( str(runset), 'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState)) if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failUnless(self.__isCompListRunning(compList, runNum), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logger.addExpectedExact("0 physics events collected in 0 seconds") logger.addExpectedExact("0 moni events, 0 SN events, 0 tcals") logger.addExpectedExact("Run terminated SUCCESSFULLY.") self.failIf(runset.stopRun(), "stopRun() encountered error") expState = "ready" self.assertEqual( str(runset), 'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState)) if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) runset.reset() expState = "idle" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) if len(compList) > 0: self.failIf(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10)
def buildRunset(self, nodeList): if LOUD: print '-- Nodes' for node in nodeList: print node.getDescription() nodeLog = {} pool = DAQPool() port = -1 for node in nodeList: key = '%s#%d' % (node.name, node.num) nodeLog[key] = MockAppender('Log-%s' % key) pool.add(MockDAQClient(node.name, node.num, None, port, 0, node.getConnections(), nodeLog[key], node.outLinks)) port -= 1 if LOUD: print '-- Pool has ' + str(len(pool.pool)) + ' comps' for k in pool.pool.keys(): print ' ' + str(k) for c in pool.pool[k]: print ' ' + str(c) numComps = len(pool.pool) nameList = [] for node in nodeList: nameList.append(node.name + '#' + str(node.num)) logger = MockLogger('main') runset = pool.makeRunset(nameList, logger) chkId = ConnectionTest.EXP_ID ConnectionTest.EXP_ID += 1 self.assertEquals(len(pool.pool), 0) self.assertEquals(len(pool.sets), 1) self.assertEquals(pool.sets[0], runset) self.assertEquals(runset.id, chkId) self.assertEquals(len(runset.set), len(nodeList)) # copy node list # tmpList = nodeList[:] # validate all components in runset # for comp in runset.set: node = None for t in tmpList: if comp.name == t.name and comp.num == t.num: node = t tmpList.remove(t) break self.failIf(not node, 'Could not find component ' + str(comp)) # copy connector list # compConn = comp.connectors[:] # remove all output connectors # for typ in node.outLinks: conn = None for c in compConn: if not c.isInput() and c.type == typ: conn = c compConn.remove(c) break self.failIf(not conn, 'Could not find connector ' + typ + ' for component ' + str(comp)) # remove all input connectors # for typ in node.inLinks: conn = None for c in compConn: if c.isInput and c.type == typ: conn = c compConn.remove(c) break self.failIf(not conn, 'Could not find connector ' + typ + ' for component ' + str(comp)) # whine if any connectors are left # self.assertEquals(len(compConn), 0, 'Found extra connectors in ' + str(compConn)) # whine if any components are left # self.assertEquals(len(tmpList), 0, 'Found extra components in ' + str(tmpList)) if LOUD: print '-- SET: ' + str(runset) for key in nodeLog: nodeLog[key].addExpectedExact('End of log') pool.returnRunset(runset) self.assertEquals(len(pool.pool), numComps) self.assertEquals(len(pool.sets), 0) logger.checkStatus(10) for key in nodeLog: nodeLog[key].checkStatus(10)
def runTests(self, compList, runNum): logger = MockLogger('foo#0') num = 1 for c in compList: c.setOrder(num) num += 1 runset = RunSet(compList, logger) self.assertEqual(str(runset), 'RunSet #' + str(runset.id)) self.checkStatus(runset, compList, 'idle') logList = [] for c in compList: logList.append([c.name, c.num, 666, 'info']) runset.configureLogging('localhost', logList) if len(compList) > 0: self.failIf(self.isCompListConfigured(compList), 'Components should not be configured') self.failIf(self.isCompListRunning(compList), 'Components should not be running') self.assertRaises(ValueError, runset.startRun, 1) self.assertRaises(ValueError, runset.stopRun) i = 0 while True: cfgWaitStr = None for c in compList: if c.getConfigureWait() > i: if cfgWaitStr is None: cfgWaitStr = c.getName() else: cfgWaitStr += ', ' + c.getName() if cfgWaitStr is None: break logger.addExpectedExact('RunSet #%d: Waiting for configuring: %s' % (runset.id, cfgWaitStr)) i += 1 runset.configure('xxx') self.assertEqual(str(runset), 'RunSet #' + str(runset.id)) if len(compList) > 0: self.failUnless(self.isCompListConfigured(compList), 'Components should be configured') self.failIf(self.isCompListRunning(compList), 'Components should not be running') self.checkStatus(runset, compList, 'ready') self.assertRaises(ValueError, runset.stopRun) runset.startRun(runNum) self.assertEqual(str(runset), 'RunSet #' + str(runset.id) + ' run#' + str(runNum)) if len(compList) > 0: self.failUnless(self.isCompListConfigured(compList), 'Components should be configured') self.failUnless(self.isCompListRunning(compList, runNum), 'Components should not be running') self.checkStatus(runset, compList, 'running') runset.stopRun() self.assertEqual(str(runset), 'RunSet #' + str(runset.id)) if len(compList) > 0: self.failUnless(self.isCompListConfigured(compList), 'Components should be configured') self.failIf(self.isCompListRunning(compList), 'Components should not be running') self.checkStatus(runset, compList, 'ready') runset.reset() self.assertEqual(str(runset), 'RunSet #' + str(runset.id)) if len(compList) > 0: self.failIf(self.isCompListConfigured(compList), 'Components should be configured') self.failIf(self.isCompListRunning(compList), 'Components should not be running') self.checkStatus(runset, compList, 'idle') logger.checkStatus(10)
def testStartRun(self): self.__runConfigDir = tempfile.mkdtemp() mgr = MyDAQPool() a = MockComponent('aHub', 0) a.addOutput('ab') b = MockComponent('b', 0) b.addInput('ab', 123) b.addOutput('bc') c = MockComponent('c', 0) c.addInput('bc', 456) compList = [c, a, b] self.assertEqual(mgr.numComponents(), 0) for c in compList: mgr.add(c) self.assertEqual(mgr.numComponents(), len(compList)) runConfig = self.__createRunConfigFile(compList) logger = MockLogger('main') logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig) logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig) logger.addExpectedRegexp("Built runset #\d+: .*") runset = mgr.makeRunset(self.__runConfigDir, runConfig, 0, logger, forceRestart=False, strict=False) self.assertEqual(mgr.numComponents(), 0) self.assertEqual(runset.size(), len(compList)) self.__checkRunsetState(runset, 'ready') clusterName = "cluster-foo" dashLog = runset.getLog("dashLog") dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" + r" \d+\S*") dashLog.addExpectedExact("Run configuration: %s" % runConfig) dashLog.addExpectedExact("Cluster configuration: %s" % clusterName) self.__checkRunsetState(runset, 'ready') runNum = 1 moniType = RunOption.MONI_TO_NONE logger.addExpectedExact("Starting run #%d with \"%s\"" % (runNum, clusterName)) dashLog.addExpectedExact("Starting run %d..." % runNum) global ACTIVE_WARNING if not LIVE_IMPORT and not ACTIVE_WARNING: ACTIVE_WARNING = True dashLog.addExpectedExact("Cannot import IceCube Live code, so" + " per-string active DOM stats wil not" + " be reported") versionInfo = { "filename": "fName", "revision": "1234", "date": "date", "time": "time", "author": "author", "release": "rel", "repo_rev": "1repoRev", } spadeDir = "/tmp" copyDir = None runset.startRun(runNum, clusterName, moniType, versionInfo, spadeDir, copyDir) self.__checkRunsetState(runset, 'running') numEvts = 0 numSecs = 0 numMoni = 0 numSN = 0 numTcals = 0 dashLog.addExpectedExact("%d physics events collected in %d seconds" % (numEvts, numSecs)) dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" % (numMoni, numSN, numTcals)) dashLog.addExpectedExact("Run terminated SUCCESSFULLY.") self.failIf(runset.stopRun(), "stopRun() encountered error") self.__checkRunsetState(runset, 'ready') mgr.returnRunset(runset, logger) self.assertEqual(runset.id(), None) self.assertEqual(runset.configured(), False) self.assertEqual(runset.runNumber(), None) self.assertEqual(mgr.numComponents(), len(compList)) self.assertEqual(runset.size(), 0) logger.checkStatus(10)
def testSetUpLoggers(self): runNum = 9753 dr = MostlyDAQRun() dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR) logger = MockLogger('main') dr.log = logger expId = 99 expComps = [(3, 'foon', 5, 'localhost', 1234, 5678), (7, 'barn', 4, 'localhost', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dr.fill_component_dictionaries(cnc) logger.addExpectedExact('Setting up logging for %d components' % len(expComps)) nextPort = DAQPort.RUNCOMP_BASE for i in range(0, len(expComps)): c = expComps[i] logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' % (c[1], c[0], c[3], c[4], dr.ip, nextPort)) nextPort += 1 try: dr.setup_component_loggers(cnc, 'xxx', expId) finally: for k in dr.loggerOf.keys(): if dr.loggerOf[k] is not None: dr.loggerOf[k].stopServing() for c in expComps: path = os.path.join(dr.getLogPath(), '%s-%d.log' % (c[1], c[2])) if os.path.exists(path): os.remove(path) os.rmdir(dr.getLogPath()) logList = cnc.getRunsetLoggers() if logList is None: self.fail('Runset logging was not set') self.assertEquals(len(expComps), len(logList), 'Expected %d loggers, not %d' % (len(expComps), len(logList))) nextPort = DAQPort.RUNCOMP_BASE for i in range(0, len(expComps)): c = expComps[i] l = logList[i] self.assertEquals(c[1], l[0], 'Expected short name #%d "%s", not "%s"' % (i, c[1], l[0])) self.assertEquals(c[2], l[1], 'Expected DAQ ID #%d %d, not %d' % (i, c[2], l[1])) self.assertEquals(nextPort, l[2], 'Expected log port #%d %d, not %d' % (i, nextPort, l[2])) nextPort += 1 logger.checkStatus(10)
def testRunIndirect(self): self.__copyDir = tempfile.mkdtemp() self.__runConfigDir = tempfile.mkdtemp() self.__spadeDir = tempfile.mkdtemp() comps = [ MockComponent("stringHub", self.HUB_NUMBER, (MockConn("stringHit", "o"), )), MockComponent("inIceTrigger", conn=(MockConn("stringHit", "i"), MockConn("trigger", "o"))), MockComponent("globalTrigger", conn=(MockConn("trigger", "i"), MockConn("glblTrig", "o"))), MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )), MockComponent("extraComp") ] cluCfg = MockClusterConfig("clusterFoo") for comp in comps: cluCfg.addComponent(comp.fullName(), "java", "", "localhost") self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg, copyDir=self.__copyDir, runConfigDir=self.__runConfigDir, spadeDir=self.__spadeDir) catchall = self.__cnc.getLogServer() self.__loadBeanData(comps) self.__loadRadarDOMMap() nameList = [] for c in comps: self.__cnc.add(c) if c.name() != "stringHub" and c.name() != "extraComp": nameList.append(str(c)) runCompList = [] for c in comps: if c.isSource() or c.name() == "extraComp": continue runCompList.append(c.fullName()) domList = [ MockRunConfigFile.createDOM(self.RADAR_DOM), ] rcFile = MockRunConfigFile(self.__runConfigDir) runConfig = rcFile.create(runCompList, domList) catchall.addExpectedText("Loading run configuration \"%s\"" % runConfig) catchall.addExpectedText("Loaded run configuration \"%s\"" % runConfig) catchall.addExpectedTextRegexp(r"Built runset #\d+: .*") rsId = self.__cnc.rpc_runset_make(runConfig) rs = self.__cnc.findRunset(rsId) self.failIf(rs is None, "Could not find runset #%d" % rsId) time.sleep(1) if catchall: catchall.checkStatus(5) dashLog = MockLogger("dashLog") rs.setDashLog(dashLog) liveMoni = SocketReader("liveMoni", DAQPort.I3LIVE, 99) liveMoni.startServing() runNum = 345 catchall.addExpectedText("Starting run #%d with \"%s\"" % (runNum, cluCfg.configName())) dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" + " \d+\S+") dashLog.addExpectedExact("Run configuration: %s" % runConfig) dashLog.addExpectedExact("Cluster configuration: %s" % cluCfg.configName()) dashLog.addExpectedExact("Starting run %d..." % runNum) global ACTIVE_WARNING if not LIVE_IMPORT and not ACTIVE_WARNING: ACTIVE_WARNING = True dashLog.addExpectedExact("Cannot import IceCube Live code, so" + " per-string active DOM stats wil not" + " be reported") self.__cnc.rpc_runset_start_run(rsId, runNum, RunOption.MONI_TO_LIVE) if catchall: catchall.checkStatus(5) dashLog.checkStatus(5) liveMoni.checkStatus(5) numEvts = 5 payTime = 50000000001 firstTime = 1 self.__checkRateTask(comps, rs, liveMoni, dashLog, numEvts, payTime, firstTime, runNum) self.__checkMonitorTask(comps, rs, liveMoni) self.__checkActiveDOMsTask(comps, rs, liveMoni) self.__checkWatchdogTask(comps, rs, dashLog) self.__checkRadarTask(comps, rs, liveMoni) if catchall: catchall.checkStatus(5) dashLog.checkStatus(5) liveMoni.checkStatus(5) numMoni = 0 numSN = 0 numTcals = 0 duration = self.__computeDuration(firstTime, payTime) if duration <= 0: hzStr = "" else: hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration) dashLog.addExpectedExact( "%d physics events collected in %d seconds%s" % (numEvts, duration, hzStr)) dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" % (numMoni, numSN, numTcals)) dashLog.addExpectedExact("Run terminated SUCCESSFULLY.") self.__addRunStopMoni(liveMoni, payTime, numEvts, runNum) self.__cnc.rpc_runset_stop_run(rsId) time.sleep(1) if catchall: catchall.checkStatus(5) dashLog.checkStatus(5) liveMoni.checkStatus(5) self.__cnc.rpc_runset_break(rsId) if catchall: catchall.checkStatus(5) dashLog.checkStatus(5) liveMoni.checkStatus(5) catchall.stopServing() liveMoni.stopServing()