def __runSubrun(self, compList, runNum, spadeDir="/tmp", copyDir=None, expectError=None): logger = MockLogger('LOG') num = 1 for c in compList: c.setOrder(num) num += 1 runConfig = FakeRunConfig("XXXrunSubXXX") clusterName = "cluster-foo" runset = MyRunSet(MyParent(), runConfig, compList, logger) expState = "idle" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) runset.configure() expState = "ready" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logDir = "/tmp" #runset.startLogging(logDir, 123, clusterName) if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logger.addExpectedRegexp("Could not stop run: .*") try: stopErr = runset.stopRun() except RunSetException, ve: if not "is not running" in str(ve): raise ve stopErr = False
def testBuildMultiInput(self): self.__runConfigDir = tempfile.mkdtemp() mgr = MyDAQPool() compList = [] comp = MockComponent('fooHub', 0) comp.addOutput('conn') compList.append(comp) comp = MockComponent('bar', 0) comp.addInput('conn', 123) compList.append(comp) comp = MockComponent('baz', 0) comp.addInput('conn', 456) compList.append(comp) self.assertEqual(mgr.numComponents(), 0) for c in compList: mgr.add(c) self.assertEqual(mgr.numComponents(), len(compList)) runConfig = self.__createRunConfigFile(compList) logger = MockLogger('main') logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig) logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig) logger.addExpectedRegexp("Built runset #\d+: .*") runset = mgr.makeRunset(self.__runConfigDir, runConfig, 0, logger, forceRestart=False, strict=False) self.assertEqual(mgr.numComponents(), 0) found = mgr.findRunset(runset.id()) self.failIf(found is None, "Couldn't find runset #%d" % runset.id()) mgr.returnRunset(runset, logger) self.assertEqual(mgr.numComponents(), len(compList)) for c in compList: mgr.remove(c) self.assertEqual(mgr.numComponents(), 0) logger.checkStatus(10)
def testRunsetBreakBad(self): dr = MostlyDAQRun() logger = MockLogger('main') dr.log = logger expId = 99 expComps = [(3, 'foo', 0, 'localhost', 1234, 5678), (7, 'bar', 1, 'localhost', 4321, 8765)] cnc = MockCnCRPC() cnc.setRunSet(expId, expComps) cnc.denyBreak = True dr.runSetID = expId logger.addExpectedExact('Breaking run set...') logger.addExpectedRegexp('WARNING: failed to break run set - .*') dr.break_existing_runset(cnc) self.failIf(cnc.RSBreakFlag, 'Runset was broken') self.assertEquals(0, len(dr.setCompIDs), 'Should not have any components') self.assertEquals(0, len(dr.shortNameOf), 'Should not have any short names') self.assertEquals(0, len(dr.daqIDof), 'Should not have any DAQ IDs') self.assertEquals(0, len(dr.rpcAddrOf), 'Should not have any RPC addresses') self.assertEquals(0, len(dr.rpcPortOf), 'Should not have any RPC ports') self.assertEquals(0, len(dr.mbeanPortOf), 'Should not have any MBean ports') if dr.runSetID is not None: self.fail('Runset ID should be unset') if dr.lastConfig is not None: self.fail('Last config should be unset') logger.checkStatus(10)
def testStartRun(self): self.__runConfigDir = tempfile.mkdtemp() mgr = MyDAQPool() a = MockComponent('aHub', 0) a.addOutput('ab') b = MockComponent('b', 0) b.addInput('ab', 123) b.addOutput('bc') c = MockComponent('c', 0) c.addInput('bc', 456) compList = [c, a, b] self.assertEqual(mgr.numComponents(), 0) for c in compList: mgr.add(c) self.assertEqual(mgr.numComponents(), len(compList)) runConfig = self.__createRunConfigFile(compList) logger = MockLogger('main') logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig) logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig) logger.addExpectedRegexp("Built runset #\d+: .*") runset = mgr.makeRunset(self.__runConfigDir, runConfig, 0, logger, forceRestart=False, strict=False) self.assertEqual(mgr.numComponents(), 0) self.assertEqual(runset.size(), len(compList)) self.__checkRunsetState(runset, 'ready') clusterName = "cluster-foo" dashLog = runset.getLog("dashLog") dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" + r" \d+\S*") dashLog.addExpectedExact("Run configuration: %s" % runConfig) dashLog.addExpectedExact("Cluster configuration: %s" % clusterName) self.__checkRunsetState(runset, 'ready') runNum = 1 moniType = RunOption.MONI_TO_NONE logger.addExpectedExact("Starting run #%d with \"%s\"" % (runNum, clusterName)) dashLog.addExpectedExact("Starting run %d..." % runNum) global ACTIVE_WARNING if not LIVE_IMPORT and not ACTIVE_WARNING: ACTIVE_WARNING = True dashLog.addExpectedExact("Cannot import IceCube Live code, so" + " per-string active DOM stats wil not" + " be reported") versionInfo = { "filename": "fName", "revision": "1234", "date": "date", "time": "time", "author": "author", "release": "rel", "repo_rev": "1repoRev", } spadeDir = "/tmp" copyDir = None runset.startRun(runNum, clusterName, moniType, versionInfo, spadeDir, copyDir) self.__checkRunsetState(runset, 'running') numEvts = 0 numSecs = 0 numMoni = 0 numSN = 0 numTcals = 0 dashLog.addExpectedExact("%d physics events collected in %d seconds" % (numEvts, numSecs)) dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" % (numMoni, numSN, numTcals)) dashLog.addExpectedExact("Run terminated SUCCESSFULLY.") self.failIf(runset.stopRun(), "stopRun() encountered error") self.__checkRunsetState(runset, 'ready') mgr.returnRunset(runset, logger) self.assertEqual(runset.id(), None) self.assertEqual(runset.configured(), False) self.assertEqual(runset.runNumber(), None) self.assertEqual(mgr.numComponents(), len(compList)) self.assertEqual(runset.size(), 0) logger.checkStatus(10)
def __runTests(self, compList, runNum): logger = MockLogger('foo#0') num = 1 for c in compList: c.setOrder(num) num += 1 runConfig = FakeRunConfig("XXXrunCfgXXX") expId = RunSet.ID.peekNext() clusterName = "cluster-foo" spadeDir = "/tmp" copyDir = None runset = MyRunSet(MyParent(), runConfig, compList, logger) expState = "idle" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) expState = "configuring" i = 0 while True: cfgWaitStr = None for c in compList: if c.getConfigureWait() > i: if cfgWaitStr is None: cfgWaitStr = c.fullName() else: cfgWaitStr += ', ' + c.fullName() if cfgWaitStr is None: break logger.addExpectedExact("RunSet #%d (%s): Waiting for %s: %s" % (expId, expState, expState, cfgWaitStr)) i += 1 runset.configure() expState = "ready" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logDir = "/tmp" expState = "ready" if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logger.addExpectedRegexp("Could not stop run: .*") self.assertRaises(RunSetException, runset.stopRun) logger.checkStatus(10) versionInfo = { "filename": "fName", "revision": "1234", "date": "date", "time": "time", "author": "author", "release": "rel", "repo_rev": "1repoRev", } expState = "running" global CAUGHT_WARNING if not LIVE_IMPORT and not CAUGHT_WARNING: CAUGHT_WARNING = True logger.addExpectedRegexp(r"^Cannot import IceCube Live.*") logger.addExpectedExact("Starting run #%d with \"%s\"" % (runNum, clusterName)) logger.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" + r" \d+\S*") logger.addExpectedExact("Cluster configuration: %s" % clusterName) logger.addExpectedExact("Run configuration: %s" % runConfig.basename()) logger.addExpectedExact("Starting run %d..." % runNum) runset.startRun(runNum, clusterName, RunOption.MONI_TO_NONE, versionInfo, spadeDir, copyDir, logDir) self.assertEqual( str(runset), 'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState)) if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failUnless(self.__isCompListRunning(compList, runNum), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) logger.addExpectedExact("0 physics events collected in 0 seconds") logger.addExpectedExact("0 moni events, 0 SN events, 0 tcals") logger.addExpectedExact("Run terminated SUCCESSFULLY.") self.failIf(runset.stopRun(), "stopRun() encountered error") expState = "ready" self.assertEqual( str(runset), 'RunSet #%d run#%d (%s)' % (runset.id(), runNum, expState)) if len(compList) > 0: self.failUnless(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10) runset.reset() expState = "idle" self.assertEqual(str(runset), 'RunSet #%d (%s)' % (runset.id(), expState)) if len(compList) > 0: self.failIf(self.__isCompListConfigured(compList), 'Components should be configured') self.failIf(self.__isCompListRunning(compList), 'Components should not be running') self.__checkStatus(runset, compList, expState) logger.checkStatus(10)
def testRunIndirect(self): self.__copyDir = tempfile.mkdtemp() self.__runConfigDir = tempfile.mkdtemp() self.__spadeDir = tempfile.mkdtemp() comps = [ MockComponent("stringHub", self.HUB_NUMBER, (MockConn("stringHit", "o"), )), MockComponent("inIceTrigger", conn=(MockConn("stringHit", "i"), MockConn("trigger", "o"))), MockComponent("globalTrigger", conn=(MockConn("trigger", "i"), MockConn("glblTrig", "o"))), MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )), MockComponent("extraComp") ] cluCfg = MockClusterConfig("clusterFoo") for comp in comps: cluCfg.addComponent(comp.fullName(), "java", "", "localhost") self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg, copyDir=self.__copyDir, runConfigDir=self.__runConfigDir, spadeDir=self.__spadeDir) catchall = self.__cnc.getLogServer() self.__loadBeanData(comps) self.__loadRadarDOMMap() nameList = [] for c in comps: self.__cnc.add(c) if c.name() != "stringHub" and c.name() != "extraComp": nameList.append(str(c)) runCompList = [] for c in comps: if c.isSource() or c.name() == "extraComp": continue runCompList.append(c.fullName()) domList = [ MockRunConfigFile.createDOM(self.RADAR_DOM), ] rcFile = MockRunConfigFile(self.__runConfigDir) runConfig = rcFile.create(runCompList, domList) catchall.addExpectedText("Loading run configuration \"%s\"" % runConfig) catchall.addExpectedText("Loaded run configuration \"%s\"" % runConfig) catchall.addExpectedTextRegexp(r"Built runset #\d+: .*") rsId = self.__cnc.rpc_runset_make(runConfig) rs = self.__cnc.findRunset(rsId) self.failIf(rs is None, "Could not find runset #%d" % rsId) time.sleep(1) if catchall: catchall.checkStatus(5) dashLog = MockLogger("dashLog") rs.setDashLog(dashLog) liveMoni = SocketReader("liveMoni", DAQPort.I3LIVE, 99) liveMoni.startServing() runNum = 345 catchall.addExpectedText("Starting run #%d with \"%s\"" % (runNum, cluCfg.configName())) dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" + " \d+\S+") dashLog.addExpectedExact("Run configuration: %s" % runConfig) dashLog.addExpectedExact("Cluster configuration: %s" % cluCfg.configName()) dashLog.addExpectedExact("Starting run %d..." % runNum) global ACTIVE_WARNING if not LIVE_IMPORT and not ACTIVE_WARNING: ACTIVE_WARNING = True dashLog.addExpectedExact("Cannot import IceCube Live code, so" + " per-string active DOM stats wil not" + " be reported") self.__cnc.rpc_runset_start_run(rsId, runNum, RunOption.MONI_TO_LIVE) if catchall: catchall.checkStatus(5) dashLog.checkStatus(5) liveMoni.checkStatus(5) numEvts = 5 payTime = 50000000001 firstTime = 1 self.__checkRateTask(comps, rs, liveMoni, dashLog, numEvts, payTime, firstTime, runNum) self.__checkMonitorTask(comps, rs, liveMoni) self.__checkActiveDOMsTask(comps, rs, liveMoni) self.__checkWatchdogTask(comps, rs, dashLog) self.__checkRadarTask(comps, rs, liveMoni) if catchall: catchall.checkStatus(5) dashLog.checkStatus(5) liveMoni.checkStatus(5) numMoni = 0 numSN = 0 numTcals = 0 duration = self.__computeDuration(firstTime, payTime) if duration <= 0: hzStr = "" else: hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration) dashLog.addExpectedExact( "%d physics events collected in %d seconds%s" % (numEvts, duration, hzStr)) dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" % (numMoni, numSN, numTcals)) dashLog.addExpectedExact("Run terminated SUCCESSFULLY.") self.__addRunStopMoni(liveMoni, payTime, numEvts, runNum) self.__cnc.rpc_runset_stop_run(rsId) time.sleep(1) if catchall: catchall.checkStatus(5) dashLog.checkStatus(5) liveMoni.checkStatus(5) self.__cnc.rpc_runset_break(rsId) if catchall: catchall.checkStatus(5) dashLog.checkStatus(5) liveMoni.checkStatus(5) catchall.stopServing() liveMoni.stopServing()
def __runDirect(self, failReset): self.__copyDir = tempfile.mkdtemp() self.__runConfigDir = tempfile.mkdtemp() self.__spadeDir = tempfile.mkdtemp() comps = [ MockComponent("stringHub", self.HUB_NUMBER, (MockConn("stringHit", "o"), )), MockComponent("inIceTrigger", conn=(MockConn("stringHit", "i"), MockConn("trigger", "o"))), MockComponent("globalTrigger", conn=(MockConn("trigger", "i"), MockConn("glblTrig", "o"))), MockComponent("eventBuilder", conn=(MockConn("glblTrig", "i"), )), MockComponent("extraComp") ] cluCfg = MockClusterConfig("clusterFoo") for comp in comps: cluCfg.addComponent(comp.fullName(), "java", "", "localhost") self.__cnc = MostlyCnCServer(clusterConfigObject=cluCfg) self.__loadBeanData(comps) nameList = [] for c in comps: self.__cnc.add(c) if c.name() != "stringHub" and c.name() != "extraComp": nameList.append(str(c)) domList = [ MockRunConfigFile.createDOM(self.RADAR_DOM), ] rcFile = MockRunConfigFile(self.__runConfigDir) runConfig = rcFile.create(nameList, domList) logger = MockLogger("main") logger.addExpectedExact("Loading run configuration \"%s\"" % runConfig) logger.addExpectedExact("Loaded run configuration \"%s\"" % runConfig) logger.addExpectedRegexp(r"Built runset #\d+: .*") rs = self.__cnc.makeRunset(self.__runConfigDir, runConfig, 0, logger, forceRestart=False, strict=False) logger.checkStatus(5) dashLog = MockLogger("dashLog") rs.setDashLog(dashLog) runNum = 321 logger.addExpectedExact("Starting run #%d with \"%s\"" % (runNum, cluCfg.configName())) dashLog.addExpectedRegexp(r"Version info: \S+ \d+ \S+ \S+ \S+ \S+" + " \d+\S+") dashLog.addExpectedExact("Run configuration: %s" % runConfig) dashLog.addExpectedExact("Cluster configuration: %s" % cluCfg.configName()) dashLog.addExpectedExact("Starting run %d..." % runNum) global ACTIVE_WARNING if not LIVE_IMPORT and not ACTIVE_WARNING: ACTIVE_WARNING = True dashLog.addExpectedExact("Cannot import IceCube Live code, so" + " per-string active DOM stats wil not" + " be reported") versionInfo = { "filename": "fName", "revision": "1234", "date": "date", "time": "time", "author": "author", "release": "rel", "repo_rev": "1repoRev", } rs.startRun(runNum, cluCfg.configName(), RunOption.MONI_TO_NONE, versionInfo, "/tmp") logger.checkStatus(5) dashLog.checkStatus(5) numEvts = 1000 payTime = 50000000001 firstTime = 1 self.__checkRateTask(comps, rs, None, dashLog, numEvts, payTime, firstTime, runNum) numMoni = 0 numSN = 0 numTcals = 0 duration = self.__computeDuration(firstTime, payTime) if duration <= 0: hzStr = "" else: hzStr = " (%2.2f Hz)" % self.__computeRateHz(0, numEvts, duration) dashLog.addExpectedExact( "%d physics events collected in %d seconds%s" % (numEvts, duration, hzStr)) dashLog.addExpectedExact("%d moni events, %d SN events, %d tcals" % (numMoni, numSN, numTcals)) dashLog.addExpectedExact("Run terminated SUCCESSFULLY.") self.failIf(rs.stopRun(), "stopRun() encountered error") logger.checkStatus(5) dashLog.checkStatus(5) if failReset: rs.setUnresetComponent(comps[0]) logger.addExpectedExact("Restarting %s (state '%s' after reset)" % (comps[0], MyRunSet.FAIL_STATE)) logger.addExpectedExact("Cycling components [%s]" % comps[0]) try: self.__cnc.returnRunset(rs, logger) if failReset: self.fail("returnRunset should not have succeeded") except RunSetException: if not failReset: raise logger.checkStatus(5) dashLog.checkStatus(5)