def testCheckAllWatchdogMax(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger dr.watchdog = MockWatchdog() dr.watchdog.inProg = True dr.watchdog.done = True DAQRun.unHealthyCount = DAQRun.MAX_UNHEALTHY_COUNT expCnt = 0 rtnVal = dr.check_all() self.failIf(rtnVal, 'Expected call to succeed') self.failUnless(dr.watchdog.threadCleared, 'Should have cleared thread') self.failIf(dr.watchdog.watchStarted, 'Should not have started watchdog') self.assertEquals(expCnt, DAQRun.unHealthyCount, 'UnhealthyCount should be %d, not %d' % (expCnt, DAQRun.unHealthyCount)) logger.checkStatus(10)
def testWaitForRequiredBad(self): comps = [(0, 'abc', 1, 'xxx', 1, 2), (1, 'def', 2, 'yyy', 3, 4), (2, 'ghi', 3, 'zzz', 5, 6)] dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger cnc = MockCnCRPC() cnc.setComponents(comps[1:]) expRunNum = 100 expId = 123 dr.runStats.runNum = expRunNum dr.runSetID = expId DAQRun.COMP_TOUT = 0 required = [] for c in comps: required.append('%s#%d' % (c[1], c[2])) logger.addExpectedExact(('Starting run %d (waiting for required %d' + ' components to register w/ CnCServer)') % (dr.runStats.runNum, len(required))) try: dr.build_run_set(cnc, required) self.fail('Unexpected success') except Exception, e: self.assertEquals('Still waiting for ' + required[0], str(e), 'Unexpected exception message "%s"' % str(e))
def testBuildMissingOneOutput(self): mgr = DAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addOutput('aaa') comp.addInput('xxx', 123) compList.append(comp) comp = MockComponent('bar', 0) comp.addInput('aaa', 456) compList.append(comp) self.assertEqual(len(mgr.pool), 0) nameList = [] for c in compList: mgr.add(c) nameList.append(c.name) self.assertEqual(len(mgr.pool), len(compList)) logger = MockLogger('main') self.assertRaises(ValueError, mgr.makeRunset, nameList, logger) self.assertEqual(len(mgr.pool), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(len(mgr.pool), 0) logger.checkStatus(10)
def __runSubrun(self, compList, runNum, spadeDir="/tmp", copyDir=None, expectError=None): logger = MockLogger('LOG') num = 1 for c in compList: c.setOrder(num) num += 1 runConfig = FakeRunConfig("XXXrunSubXXX") clusterName = "cluster-foo" runset = MyRunSet(MyParent(), runConfig, compList, logger) expState = "idle" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) runset.configure() expState = "ready" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logDir = "/tmp" #runset.startLogging(logDir, 123, clusterName) if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logger.addExpectedRegexp("Could not stop run: .*") try: stopErr = runset.stopRun() except RunSetException, ve: if not "is not running" in str(ve): raise ve stopErr = False
def testVersion(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) self.assertEqual(live.version(), MockCnC.RELEASE + "_" + MockCnC.REPO_REV) log.checkStatus(1)
def testCheckAllNone(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger rtnVal = dr.check_all() self.failUnless(rtnVal, 'Expected call to succeed') logger.checkStatus(10)
def testBuildMultiInput(self): self.__runConfigDir = tempfile.mkdtemp() mgr = MyDAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addOutput('conn') compList.append(comp) comp = MockComponent('bar', 0) comp.addInput('conn', 123) compList.append(comp) comp = MockComponent('baz', 0) comp.addInput('conn', 456) compList.append(comp) self.assertEqual(mgr.numComponents(), 0) for c in compList: mgr.add(c) self.assertEqual(mgr.numComponents(), len(compList)) runConfig = self.__createRunConfigFile(compList) logger = MockLogger('main') logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig) logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig) logger.addExpectedRegexp("Built runset #\d+: .*") runset = mgr.makeRunset(self.__runConfigDir, runConfig, 0, logger, forceRestart=False, strict=False) self.assertEqual(mgr.numComponents(), 0) found = mgr.findRunset(runset.id()) self.failIf(found is None, "Couldn't find runset #%d" % runset.id()) mgr.returnRunset(runset, logger) self.assertEqual(mgr.numComponents(), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(mgr.numComponents(), 0) logger.checkStatus(10)
def testRPCStopRunSuccess(self): dr = StubbedDAQRun() logger = MockLogger('main') dr.log = logger dr.runState = 'RUNNING' dr.rpc_stop_run() self.assertEquals('STOPPING', dr.runState, 'Should be stopping, not ' + dr.runState) logger.checkStatus(10)
def testStartingNoStateArgs(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) self.assertRaisesMsg(LiveException("No stateArgs specified"), live.starting, None)
def testRunningNothing(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) exc = LiveException("Cannot check run state; no active runset") self.assertRaisesMsg(exc, live.running)
def testStoppingNoRunset(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) exc = LiveException("Cannot stop run; no active runset") self.assertRaisesMsg(exc, live.stopping)
def testBuildMultiMissing(self): mgr = DAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addInput('xxx', 123) compList.append(comp) comp = MockComponent('bar', 0) comp.addOutput('xxx') compList.append(comp) comp = MockComponent('feeHub', 0) comp.addInput('xxx', 456) compList.append(comp) comp = MockComponent('baz', 0) comp.addOutput('xxx') compList.append(comp) self.assertEqual(len(mgr.pool), 0) nameList = [] for c in compList: mgr.add(c) nameList.append(c.name) self.assertEqual(len(mgr.pool), len(compList)) logger = MockLogger('main') try: mgr.makeRunset(nameList, logger) self.fail('Unexpected success') except ValueError: pass except: self.fail('Unexpected exception') self.assertEqual(len(mgr.pool), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(len(mgr.pool), 0) logger.checkStatus(10)
def testBuildMultiInput(self): mgr = DAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addOutput('conn') compList.append(comp) comp = MockComponent('bar', 0) comp.addInput('conn', 123) compList.append(comp) comp = MockComponent('baz', 0) comp.addInput('conn', 456) compList.append(comp) self.assertEqual(len(mgr.pool), 0) nameList = [] for c in compList: mgr.add(c) nameList.append(c.name) self.assertEqual(len(mgr.pool), len(compList)) logger = MockLogger('main') runset = mgr.makeRunset(nameList, logger) self.assertEqual(len(mgr.pool), 0) found = mgr.findRunset(runset.id) self.failIf(found is None, "Couldn't find runset #" + str(runset.id)) mgr.returnRunset(runset) self.assertEqual(len(mgr.pool), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(len(mgr.pool), 0) logger.checkStatus(10)
def testGetEventCounts(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger numEvts = 17 numMoni = 222 numSN = 51 numTCal = 93 dr.moni = MockMoni() dr.moni.addEntry(5, 'backEnd', 'NumEventsSent', str(numEvts)) dr.moni.addEntry(17, 'moniBuilder', 'TotalDispatchedData', str(numMoni)) dr.moni.addEntry(17, 'snBuilder', 'TotalDispatchedData', str(numSN)) dr.moni.addEntry(17, 'tcalBuilder', 'TotalDispatchedData', str(numTCal)) expId = 99 expComps = [(5, 'eventBuilder', 0, 'x', 1234, 5678), (17, 'secondaryBuilders', 0, 'x', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dr.fill_component_dictionaries(cnc) expCnts = (numEvts, numMoni, numSN, numTCal) cnts = dr.getEventCounts() self.assertEquals(len(expCnts), len(cnts), 'Expected %d event counts, not %d' % (len(expCnts), len(cnts))) for i in range(0, len(expCnts)): self.assertEquals(expCnts[i], cnts[i], 'Expected event count #%d to be %d, not %d' % (i, expCnts[i], cnts[i])) logger.checkStatus(10)
def testRecovering(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) runCfg = "foo" runNum = 13579 runSet = MockRunSet(runCfg) cnc.setExpectedRunConfig(runCfg) cnc.setExpectedRunNumber(runNum) cnc.setRunSet(runSet) state = { "runConfig": runCfg, "runNumber": runNum } self.failUnless(live.starting(state), "starting failed") runSet.setExpectedStopError() log.addExpectedExact("DAQLive recovered %s" % runSet) self.failUnless(live.recovering(), "recovering failed")
def testRestartFailCluCfg(self): compList = self.__buildCompList( ("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful")) runConfig = FakeRunConfig("XXXrunCfgXXX") logger = MockLogger('foo#0') runset = MyRunSet(MyParent(), runConfig, compList, logger) baseName = "failCluCfg" clusterCfg = self.__buildClusterConfig(compList[1:], baseName) logger.addExpectedExact( ("Cannot restart component %s: Not found" + " in cluster config \"%s\"") % (compList[0].fullName(), clusterCfg.configName())) errMsg = None for c in compList[1:]: if errMsg is None: errMsg = "Cycling components [" + c.fullName() else: errMsg += ", " + c.fullName() if errMsg is not None: errMsg += "]" logger.addExpectedExact(errMsg) runset.restartComponents(compList[:], clusterCfg, None, None, None, None, False, False, False)
def testRunsetBreakGood(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger expId = 99 expComps = [(3, 'foo', 0, 'localhost', 1234, 5678), (7, 'bar', 1, 'localhost', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId logger.addExpectedExact('Breaking run set...') dr.break_existing_runset(cnc) #self.failUnless(cnc.RSBreakFlag, 'Runset was not broken') self.assertEquals(0, len(dr.setCompIDs), 'Should not have any components') self.assertEquals(0, len(dr.shortNameOf), 'Should not have any short names') self.assertEquals(0, len(dr.daqIDof), 'Should not have any DAQ IDs') self.assertEquals(0, len(dr.rpcAddrOf), 'Should not have any RPC addresses') self.assertEquals(0, len(dr.rpcPortOf), 'Should not have any RPC ports') self.assertEquals(0, len(dr.mbeanPortOf), 'Should not have any MBean ports') if dr.runSetID is not None: self.fail('Runset ID should be unset') if dr.lastConfig is not None: self.fail('Last config should be unset') logger.checkStatus(10)
def testRestartExtraComp(self): compList = self.__buildCompList( ("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful")) runConfig = FakeRunConfig("XXXrunCfgXXX") logger = MockLogger('foo#0') runset = MyRunSet(MyParent(), runConfig, compList, logger) extraComp = MockComponent("queen", 10) longList = compList[:] longList.append(extraComp) baseName = "failCluCfg" clusterCfg = self.__buildClusterConfig(longList, baseName) logger.addExpectedExact("Cannot remove component %s from RunSet #%d" % (extraComp.fullName(), runset.id())) errMsg = None for c in longList: if errMsg is None: errMsg = "Cycling components [" + c.fullName() else: errMsg += ", " + c.fullName() if errMsg is not None: errMsg += "]" logger.addExpectedExact(errMsg) runset.restartComponents(longList, clusterCfg, None, None, None, None, False, False, False)
def testFillCompDict(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger expId = 99 expComps = [(3, 'foo', 0, 'localhost', 1234, 5678), (7, 'bar', 1, 'localhost', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dr.fill_component_dictionaries(cnc) for i in range(0, len(expComps)): key = dr.setCompIDs[i] self.assertEquals(expComps[i][0], key, 'Expected comp#%d to be %s, not %s' % (i, expComps[i][0], key)) self.assertEquals(expComps[i][1], dr.shortNameOf[key], 'Expected shortName#%d to be %s, not %s' % (i, expComps[i][1], dr.shortNameOf[key])) self.assertEquals(expComps[i][2], dr.daqIDof[key], 'Expected daqID#%d to be %d, not %d' % (i, expComps[i][2], dr.daqIDof[key])) self.assertEquals(expComps[i][3], dr.rpcAddrOf[key], 'Expected rpcAddr#%d to be %s, not %s' % (i, expComps[i][3], dr.rpcAddrOf[key])) self.assertEquals(expComps[i][4], dr.rpcPortOf[key], 'Expected rpcPort#%d to be %d, not %d' % (i, expComps[i][4], dr.rpcPortOf[key])) self.assertEquals(expComps[i][5], dr.mbeanPortOf[key], 'Expected mbeanPort#%d to be %d, not %d' % (i, expComps[i][5], dr.mbeanPortOf[key])) logger.checkStatus(10)
def testRunSummaryFauxTest(self): # this is a blatant attempt to increase the code coverage, because the # rpc_run_summary method is Anvil-centric and I3Live is coming soon dr = StubbedDAQRun() logger = MockLogger('main') dr.log = logger dr.runState = "RUNNING" dr.runStats.runNum = 15 dr.prevRunStats = dr.runStats dr.rpc_daq_summary_xml()
def testRestartAll(self): compList = self.__buildCompList( ("sleepy", "sneezy", "happy", "grumpy", "doc", "dopey", "bashful")) runConfig = FakeRunConfig("XXXrunCfgXXX") logger = MockLogger('foo#0') runset = MyRunSet(MyParent(), runConfig, compList, logger) clusterCfg = self.__buildClusterConfig(compList, "restartAll") errMsg = None for c in compList: if errMsg is None: errMsg = "Cycling components [" + c.fullName() else: errMsg += ", " + c.fullName() if errMsg is not None: errMsg += "]" logger.addExpectedExact(errMsg) runset.restartAllComponents(clusterCfg, None, None, None, None, False, False, False)
def testStartingNoKeys(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) runCfg = "foo" runNum = 13579 cnc.setExpectedRunConfig(runCfg) cnc.setExpectedRunNumber(runNum) state = { } self.assertRaisesMsg(LiveException("No stateArgs specified"), live.starting, state)
def testStartingNoRunNumKey(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) runCfg = "foo" runNum = 13579 cnc.setExpectedRunConfig(runCfg) cnc.setExpectedRunNumber(runNum) state = { "runConfig": runCfg, } exc = LiveException("stateArgs does not contain key \"runNumber\"") self.assertRaisesMsg(exc, live.starting, state)
def testStarting(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) runCfg = "foo" runNum = 13579 cnc.setExpectedRunConfig(runCfg) cnc.setExpectedRunNumber(runNum) cnc.setRunSet(MockRunSet(runCfg)) state = { "runConfig": runCfg, "runNumber": runNum } self.failUnless(live.starting(state), "starting failed")
def testStartingNoRunSet(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) runCfg = "foo" runNum = 13579 cnc.setExpectedRunConfig(runCfg) cnc.setExpectedRunNumber(runNum) state = { "runConfig": runCfg, "runNumber": runNum } self.assertRaisesMsg(LiveException("Cannot create runset for \"%s\"" % runCfg), live.starting, state)
def testRPCStopRunStopped(self): dr = StubbedDAQRun() logger = MockLogger('main') dr.log = logger logger.addExpectedExact('Warning: run is already stopped.') dr.runState = 'STOPPED' dr.rpc_stop_run() logger.checkStatus(10)
def testSubrun(self): cnc = MockCnC() log = MockLogger("liveLog") live = self.__createLive(cnc, log) runCfg = "foo" runNum = 13579 runSet = MockRunSet(runCfg) cnc.setExpectedRunConfig(runCfg) cnc.setExpectedRunNumber(runNum) cnc.setRunSet(runSet) state = { "runConfig": runCfg, "runNumber": runNum } self.failUnless(live.starting(state), "starting failed") self.assertEquals("OK", live.subrun(1, ["domA", "dom2", ]))
def testSetUpAllLoggers(self): runNum = 5432 dr = MostlyDAQRun() dr.createRunLogDirectory(runNum, TestDAQRun.LOG_DIR) logger = MockLogger('main') dr.log = logger expId = 99 expComps = [(3, 'foo', 0, 'localhost', 1234, 5678), (7, 'bar', 1, 'localhost', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) dr.runSetID = expId dr.fill_component_dictionaries(cnc) nextPort = DAQPort.RUNCOMP_BASE logger.addExpectedExact('Setting up logging for %d components' % len(expComps)) for c in expComps: logger.addExpectedExact('%s(%d %s:%d) -> %s:%d' % (c[1], c[0], c[3], c[4], dr.ip, nextPort)) nextPort += 1 try: dr.setUpAllComponentLoggers() finally: for k in dr.loggerOf.keys(): if dr.loggerOf[k] is not None: dr.loggerOf[k].stopServing() for c in expComps: path = os.path.join(dr.getLogPath(), '%s-%d.log' % (c[1], c[2])) if os.path.exists(path): os.remove(path) os.rmdir(dr.getLogPath()) logger.checkStatus(10)
def testBuildMultiMissing(self): self.__runConfigDir = tempfile.mkdtemp() mgr = MyDAQPool() compList = [] outputName = "xxx" comp = MockComponent('fooHub', 0) comp.addInput(outputName, 123) compList.append(comp) comp = MockComponent('bar', 0) comp.addOutput(outputName) compList.append(comp) comp = MockComponent('feeHub', 0) comp.addInput(outputName, 456) compList.append(comp) comp = MockComponent('baz', 0) comp.addOutput(outputName) compList.append(comp) self.assertEqual(mgr.numComponents(), 0) for c in compList: mgr.add(c) self.assertEqual(mgr.numComponents(), len(compList)) runConfig = self.__createRunConfigFile(compList) logger = MockLogger('main') logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig) logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig) try: mgr.makeRunset(self.__runConfigDir, runConfig, 0, logger, forceRestart=False, strict=False) self.fail("makeRunset should not succeed") except ConnectionException, ce: if str(ce).find( "Found 2 %s inputs for 2 outputs" % outputName) < 0: raise ce
def testRunsetConfig(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger cnc = MockCnCRPC() logger.addExpectedExact('Configuring run set...') dr.runset_configure(cnc, 1, 'foo') self.failUnless(cnc.RSConfigFlag, 'Runset was not configured') logger.checkStatus(10)