def startBuild(self, buildDir=None): if buildDir: self.buildDir = buildDir os.environ['CMSINTBLD_RCDIR'] = self.buildDir self.updateTimeStamp(self.buildDir) os.environ[ "PATH"] = self.buildDir + "/cms/common:" + os.environ["PATH"] from logUpdater import LogUpdater logger = LogUpdater(self.buildDir) logger.setDryRun(self.dryRun) logger.setRelease("", self.relCycle, self.stamp) if not os.path.exists(self.cmsBuildDir): os.makedirs(self.cmsBuildDir) logDir = self.buildDir + '/' + self.relCycle + '/' + self.plat logFileName = logDir + "/fullLog" cmd = 'rm -rf ' + self.relCycle + '; mkdir -p ' + logDir + '; ' cmd += scriptPath + "/buildNightly.py " cmd += " --releasecycle " + self.relCycle + ' ' cmd += " --buildDir " + self.buildDir + ' ' if self.dryRun: cmd += ' --dryRun ' cmd += ' > ' + logFileName + ' 2>&1 ' self.doCmd(cmd, True, self.buildDir) print "log file at " + self.buildDir + "/fullLog" webLogPath = logger.webTargetDir if not os.path.exists(webLogPath): os.makedirs(webLogPath) cmd = 'cp ' + logFileName + ' ' + webLogPath self.doCmd(cmd, True) if self.dryRun: sys.stdout.flush() sys.stderr.flush() os.system('cat ' + logFileName) else: print "\n" print "web portal at:\n" print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py?rc=" + self.relCycle + "\n" print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py\n" print "+" * 80 print "\n" sys.stdout.flush() sys.stderr.flush() return
def startBuild(self, buildDir=None): if buildDir: self.buildDir = buildDir os.environ['CMSINTBLD_RCDIR'] = self.buildDir self.updateTimeStamp(self.buildDir) os.environ["PATH"] = self.buildDir+"/cms/common:"+os.environ["PATH"] from logUpdater import LogUpdater logger = LogUpdater(self.buildDir) logger.setDryRun(self.dryRun) logger.setRelease("", self.relCycle, self.stamp) if not os.path.exists(self.cmsBuildDir): os.makedirs(self.cmsBuildDir) logDir = self.buildDir+'/'+self.relCycle+'/'+self.plat logFileName = logDir+"/fullLog" cmd = 'rm -rf '+self.relCycle+'; mkdir -p '+logDir+'; ' cmd += scriptPath+"/buildNightly.py " cmd += " --releasecycle " + self.relCycle + ' ' cmd += " --buildDir " + self.buildDir + ' ' if self.dryRun : cmd += ' --dryRun ' cmd += ' > '+logFileName+' 2>&1 ' self.doCmd(cmd, True, self.buildDir) print "log file at " + self.buildDir + "/fullLog" webLogPath = logger.webTargetDir if not os.path.exists(webLogPath): os.makedirs(webLogPath) cmd = 'cp '+logFileName+' '+webLogPath self.doCmd(cmd, True) if self.dryRun : sys.stdout.flush() sys.stderr.flush() os.system('cat '+logFileName) else: print "\n" print "web portal at:\n" print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py?rc="+self.relCycle+"\n" print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py\n" print "+"*80 print "\n" sys.stdout.flush() sys.stderr.flush() return
class ReleaseTester(BuilderBase): def __init__(self, buildDir, releaseDir, appset, doInstall=True): BuilderBase.__init__(self) os.environ['STAGE_SVCCLASS']='t1transfer' os.environ['STAGE_HOST']='castorcms' os.environ['STAGER_TRACE']='3' self.doInstall=doInstall self.buildDir = buildDir self.appset = appset self.updateTimeStamp(self.buildDir) self.cmsswBuildDir = releaseDir self.release = os.path.basename(releaseDir) self.relTag = self.release self.relCycle = self.release.replace("CMSSW_","").split("_X_",1)[0].replace("_",".") day,hour = self.stamp.split('-') self.threadList = {} self.maxThreads = 2 # one for unit-tests, one for addOnTests self.doRelVal = True self.logger = None self.RelValArgs = None config.setDefaults(self.relCycle) try: self.doRelVal = config.Configuration[self.relCycle]['runRelVal'] self.RelValArgs = config.Configuration[self.relCycle]['RelValArgs'] except: pass return # -------------------------------------------------------------------------------- def setupLogger(self, doInstall=True): if not self.logger: from logUpdater import LogUpdater self.logger = LogUpdater(self.buildDir,doInstall) self.logger.setDryRun(self.dryRun) self.logger.setRelease(self.relTag, self.relCycle, self.stamp) return # -------------------------------------------------------------------------------- def getDepThreads(self, jobs=[]): deps = [] for job in jobs: if self.threadList.has_key(job) and self.threadList[job]: deps.append(self.threadList[job]) return deps # -------------------------------------------------------------------------------- def doTest(self, only=None): if not self.release : print "ReleaseTester> ERROR: no release specified !! " return self.setupLogger(self.doInstall) self.runProjectInit() if not only or 'dirsize' in only: print '\n'+80*'-'+' dirsize \n' self.threadList['dirsize'] = self.runDirSize() if not only or 'depViolation' in only: print '\n'+80*'-'+' depViolation \n' self.threadList['ddepViolation'] = self.runBuildFileDeps() if not only or 'relProducts' in only: print '\n'+80*'-'+' relProducts \n' self.threadList['relProducts'] = self.runReleaseProducts() if not only or 'addon' in only: print '\n'+80*'-'+' addOnTests \n' self.threadList['addon'] = self.runAddOnTests() if not only or 'relvalcomp' in only: print '\n'+80*'-'+' relvalcomp \n' self.threadList['relvalcomp'] = self.runBuildReports() if not only or 'unit' in only: print '\n'+80*'-'+' unit \n' self.threadList['unit'] = self.runUnitTests() if not only or 'codeRules' in only: print '\n'+80*'-'+' codeRules \n' self.threadList['codeRules'] = self.runCodeRulesChecker() if not only or 'ignominy' in only: print '\n'+80*'-'+' ignominy \n' self.threadList['ignominy'] = self.runIgnominy() if not only or 'fwbuildset' in only: print '\n'+80*'-'+' FWLite BuildSet\n' self.threadList['fwbuildset'] = self.runFWLiteBuildSet(self.getDepThreads(['ignominy'])) #if not only or 'onlbuildset' in only: # print '\n'+80*'-'+' Online BuildSet\n' # self.threadList['onlbuildset'] = self.runOnlineBuildSet(self.getDepThreads(['ignominy'])) if not only or 'dqmoffline' in only: print '\n'+80*'-'+' DQMOfflineTest \n' self.threadList['dqmoffline'] = self.runDQMOfflineTests(self.getDepThreads(['unit'])) if not only or 'relval' == only: print '\n'+80*'-'+' relval \n' self.threadList['relval'] = self.runPyRelVals(self.getDepThreads(['addon'])) if not only or 'libcheck' in only: print '\n'+80*'-'+' libcheck\n' self.threadList['libcheck'] = self.checkLibDeps() if not only or 'geom' in only: print '\n'+80*'-'+' geom \n' self.threadList['geom'] = self.runGeomTests() if not only or 'pyConfigs' in only: print '\n'+80*'-'+' pyConfigs \n' self.threadList['pyConfigs'] = self.checkPyConfigs() if not only or 'dupDict' in only: print '\n'+80*'-'+' dupDict \n' self.threadList['dupDict'] = self.runDuplicateDictCheck() print 'TestWait> waiting for tests to finish ....' for task in self.threadList: if self.threadList[task]: self.threadList[task].join() print 'TestWait> Tests finished ' return # -------------------------------------------------------------------------------- def checkPyConfigs(self, deps = []): print "Going to check python configs in ", os.getcwd() cmd = 'cd src ; '+scriptPath+'/checkPyConfigs.py > ../chkPyConf.log 2>&1' try: self.doCmd(cmd,self.dryRun,self.cmsswBuildDir) self.logger.updateLogFile("chkPyConf.log") self.logger.updateLogFile("chkPyConf.log",'testLogs') except: pass return None # -------------------------------------------------------------------------------- def checkLibDeps(self, deps = []): print "libDepTests> Going to run LibDepChk ... " thrd = None try: thrd = LibDepsTester(self.cmsswBuildDir,self.logger, deps) thrd.start() except Exception, e : print "ERROR during LibDepChk : caught exception: " + str(e) return thrd
class ReleaseTester(BuilderBase): def __init__(self, buildDir, releaseDir, appset, doInstall=True): BuilderBase.__init__(self) os.environ['STAGE_SVCCLASS'] = 't1transfer' os.environ['STAGE_HOST'] = 'castorcms' os.environ['STAGER_TRACE'] = '3' self.doInstall = doInstall self.buildDir = buildDir self.appset = appset self.updateTimeStamp(self.buildDir) self.cmsswBuildDir = releaseDir self.release = os.path.basename(releaseDir) self.relTag = self.release self.relCycle = self.release.replace("CMSSW_", "").split("_X_", 1)[0].replace("_", ".") day, hour = self.stamp.split('-') self.threadList = {} self.maxThreads = 2 # one for unit-tests, one for addOnTests self.doRelVal = True self.logger = None self.RelValArgs = None config.setDefaults(self.relCycle) try: self.doRelVal = config.Configuration[self.relCycle]['runRelVal'] self.RelValArgs = config.Configuration[self.relCycle]['RelValArgs'] except: pass return # -------------------------------------------------------------------------------- def setupLogger(self, doInstall=True): if not self.logger: from logUpdater import LogUpdater self.logger = LogUpdater(self.buildDir, doInstall) self.logger.setDryRun(self.dryRun) self.logger.setRelease(self.relTag, self.relCycle, self.stamp) return # -------------------------------------------------------------------------------- def getDepThreads(self, jobs=[]): deps = [] for job in jobs: if self.threadList.has_key(job) and self.threadList[job]: deps.append(self.threadList[job]) return deps # -------------------------------------------------------------------------------- def doTest(self, only=None): if not self.release: print "ReleaseTester> ERROR: no release specified !! " return self.setupLogger(self.doInstall) self.runProjectInit() if not only or 'dirsize' in only: print '\n' + 80 * '-' + ' dirsize \n' self.threadList['dirsize'] = self.runDirSize() if not only or 'depViolation' in only: print '\n' + 80 * '-' + ' depViolation \n' self.threadList['ddepViolation'] = self.runBuildFileDeps() if not only or 'relProducts' in only: print '\n' + 80 * '-' + ' relProducts \n' self.threadList['relProducts'] = self.runReleaseProducts() if not only or 'addon' in only: print '\n' + 80 * '-' + ' addOnTests \n' self.threadList['addon'] = self.runAddOnTests() if not only or 'relvalcomp' in only: print '\n' + 80 * '-' + ' relvalcomp \n' self.threadList['relvalcomp'] = self.runBuildReports() if not only or 'unit' in only: print '\n' + 80 * '-' + ' unit \n' self.threadList['unit'] = self.runUnitTests() if not only or 'codeRules' in only: print '\n' + 80 * '-' + ' codeRules \n' self.threadList['codeRules'] = self.runCodeRulesChecker() if not only or 'ignominy' in only: print '\n' + 80 * '-' + ' ignominy \n' self.threadList['ignominy'] = self.runIgnominy() if not only or 'fwbuildset' in only: print '\n' + 80 * '-' + ' FWLite BuildSet\n' self.threadList['fwbuildset'] = self.runFWLiteBuildSet( self.getDepThreads(['ignominy'])) #if not only or 'onlbuildset' in only: # print '\n'+80*'-'+' Online BuildSet\n' # self.threadList['onlbuildset'] = self.runOnlineBuildSet(self.getDepThreads(['ignominy'])) if not only or 'dqmoffline' in only: print '\n' + 80 * '-' + ' DQMOfflineTest \n' self.threadList['dqmoffline'] = self.runDQMOfflineTests( self.getDepThreads(['unit'])) if not only or 'relval' == only: print '\n' + 80 * '-' + ' relval \n' self.threadList['relval'] = self.runPyRelVals( self.getDepThreads(['addon'])) if not only or 'libcheck' in only: print '\n' + 80 * '-' + ' libcheck\n' self.threadList['libcheck'] = self.checkLibDeps() if not only or 'geom' in only: print '\n' + 80 * '-' + ' geom \n' self.threadList['geom'] = self.runGeomTests() if not only or 'pyConfigs' in only: print '\n' + 80 * '-' + ' pyConfigs \n' self.threadList['pyConfigs'] = self.checkPyConfigs() if not only or 'dupDict' in only: print '\n' + 80 * '-' + ' dupDict \n' self.threadList['dupDict'] = self.runDuplicateDictCheck() print 'TestWait> waiting for tests to finish ....' for task in self.threadList: if self.threadList[task]: self.threadList[task].join() print 'TestWait> Tests finished ' return # -------------------------------------------------------------------------------- def checkPyConfigs(self, deps=[]): print "Going to check python configs in ", os.getcwd() cmd = 'cd src ; ' + scriptPath + '/checkPyConfigs.py > ../chkPyConf.log 2>&1' try: self.doCmd(cmd, self.dryRun, self.cmsswBuildDir) self.logger.updateLogFile("chkPyConf.log") self.logger.updateLogFile("chkPyConf.log", 'testLogs') except: pass return None # -------------------------------------------------------------------------------- def checkLibDeps(self, deps=[]): print "libDepTests> Going to run LibDepChk ... " thrd = None try: thrd = LibDepsTester(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception, e: print "ERROR during LibDepChk : caught exception: " + str(e) return thrd