def startBuild(self, buildDir=None):
if buildDir: self.buildDir = buildDir
os.environ['CMSINTBLD_RCDIR'] = self.buildDir
self.updateTimeStamp(self.buildDir)
os.environ[
"PATH"] = self.buildDir + "/cms/common:" + os.environ["PATH"]
from logUpdater import LogUpdater
logger = LogUpdater(self.buildDir)
logger.setDryRun(self.dryRun)
logger.setRelease("", self.relCycle, self.stamp)
if not os.path.exists(self.cmsBuildDir):
os.makedirs(self.cmsBuildDir)
logDir = self.buildDir + '/' + self.relCycle + '/' + self.plat
logFileName = logDir + "/fullLog"
cmd = 'rm -rf ' + self.relCycle + '; mkdir -p ' + logDir + '; '
cmd += scriptPath + "/buildNightly.py "
cmd += " --releasecycle " + self.relCycle + ' '
cmd += " --buildDir " + self.buildDir + ' '
if self.dryRun: cmd += ' --dryRun '
cmd += ' > ' + logFileName + ' 2>&1 '
self.doCmd(cmd, True, self.buildDir)
print "log file at " + self.buildDir + "/fullLog"
webLogPath = logger.webTargetDir
if not os.path.exists(webLogPath):
os.makedirs(webLogPath)
cmd = 'cp ' + logFileName + ' ' + webLogPath
self.doCmd(cmd, True)
if self.dryRun:
sys.stdout.flush()
sys.stderr.flush()
os.system('cat ' + logFileName)
else:
print "\n"
print "web portal at:\n"
print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py?rc=" + self.relCycle + "\n"
print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py\n"
print "+" * 80
print "\n"
sys.stdout.flush()
sys.stderr.flush()
return
def startBuild(self, buildDir=None):
if buildDir: self.buildDir = buildDir
os.environ['CMSINTBLD_RCDIR'] = self.buildDir
self.updateTimeStamp(self.buildDir)
os.environ["PATH"] = self.buildDir+"/cms/common:"+os.environ["PATH"]
from logUpdater import LogUpdater
logger = LogUpdater(self.buildDir)
logger.setDryRun(self.dryRun)
logger.setRelease("", self.relCycle, self.stamp)
if not os.path.exists(self.cmsBuildDir):
os.makedirs(self.cmsBuildDir)
logDir = self.buildDir+'/'+self.relCycle+'/'+self.plat
logFileName = logDir+"/fullLog"
cmd = 'rm -rf '+self.relCycle+'; mkdir -p '+logDir+'; '
cmd += scriptPath+"/buildNightly.py "
cmd += " --releasecycle " + self.relCycle + ' '
cmd += " --buildDir " + self.buildDir + ' '
if self.dryRun : cmd += ' --dryRun '
cmd += ' > '+logFileName+' 2>&1 '
self.doCmd(cmd, True, self.buildDir)
print "log file at " + self.buildDir + "/fullLog"
webLogPath = logger.webTargetDir
if not os.path.exists(webLogPath):
os.makedirs(webLogPath)
cmd = 'cp '+logFileName+' '+webLogPath
self.doCmd(cmd, True)
if self.dryRun :
sys.stdout.flush()
sys.stderr.flush()
os.system('cat '+logFileName)
else:
print "\n"
print "web portal at:\n"
print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py?rc="+self.relCycle+"\n"
print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py\n"
print "+"*80
print "\n"
sys.stdout.flush()
sys.stderr.flush()
return
class ReleaseTester(BuilderBase):
def __init__(self, buildDir, releaseDir, appset, doInstall=True):
BuilderBase.__init__(self)
os.environ['STAGE_SVCCLASS']='t1transfer'
os.environ['STAGE_HOST']='castorcms'
os.environ['STAGER_TRACE']='3'
self.doInstall=doInstall
self.buildDir = buildDir
self.appset = appset
self.updateTimeStamp(self.buildDir)
self.cmsswBuildDir = releaseDir
self.release = os.path.basename(releaseDir)
self.relTag = self.release
self.relCycle = self.release.replace("CMSSW_","").split("_X_",1)[0].replace("_",".")
day,hour = self.stamp.split('-')
self.threadList = {}
self.maxThreads = 2 # one for unit-tests, one for addOnTests
self.doRelVal = True
self.logger = None
self.RelValArgs = None
config.setDefaults(self.relCycle)
try:
self.doRelVal = config.Configuration[self.relCycle]['runRelVal']
self.RelValArgs = config.Configuration[self.relCycle]['RelValArgs']
except:
pass
return
# --------------------------------------------------------------------------------
def setupLogger(self, doInstall=True):
if not self.logger:
from logUpdater import LogUpdater
self.logger = LogUpdater(self.buildDir,doInstall)
self.logger.setDryRun(self.dryRun)
self.logger.setRelease(self.relTag, self.relCycle, self.stamp)
return
# --------------------------------------------------------------------------------
def getDepThreads(self, jobs=[]):
deps = []
for job in jobs:
if self.threadList.has_key(job) and self.threadList[job]: deps.append(self.threadList[job])
return deps
# --------------------------------------------------------------------------------
def doTest(self, only=None):
if not self.release :
print "ReleaseTester> ERROR: no release specified !! "
return
self.setupLogger(self.doInstall)
self.runProjectInit()
if not only or 'dirsize' in only:
print '\n'+80*'-'+' dirsize \n'
self.threadList['dirsize'] = self.runDirSize()
if not only or 'depViolation' in only:
print '\n'+80*'-'+' depViolation \n'
self.threadList['ddepViolation'] = self.runBuildFileDeps()
if not only or 'relProducts' in only:
print '\n'+80*'-'+' relProducts \n'
self.threadList['relProducts'] = self.runReleaseProducts()
if not only or 'addon' in only:
print '\n'+80*'-'+' addOnTests \n'
self.threadList['addon'] = self.runAddOnTests()
if not only or 'relvalcomp' in only:
print '\n'+80*'-'+' relvalcomp \n'
self.threadList['relvalcomp'] = self.runBuildReports()
if not only or 'unit' in only:
print '\n'+80*'-'+' unit \n'
self.threadList['unit'] = self.runUnitTests()
if not only or 'codeRules' in only:
print '\n'+80*'-'+' codeRules \n'
self.threadList['codeRules'] = self.runCodeRulesChecker()
if not only or 'ignominy' in only:
print '\n'+80*'-'+' ignominy \n'
self.threadList['ignominy'] = self.runIgnominy()
if not only or 'fwbuildset' in only:
print '\n'+80*'-'+' FWLite BuildSet\n'
self.threadList['fwbuildset'] = self.runFWLiteBuildSet(self.getDepThreads(['ignominy']))
#if not only or 'onlbuildset' in only:
# print '\n'+80*'-'+' Online BuildSet\n'
# self.threadList['onlbuildset'] = self.runOnlineBuildSet(self.getDepThreads(['ignominy']))
if not only or 'dqmoffline' in only:
print '\n'+80*'-'+' DQMOfflineTest \n'
self.threadList['dqmoffline'] = self.runDQMOfflineTests(self.getDepThreads(['unit']))
if not only or 'relval' == only:
print '\n'+80*'-'+' relval \n'
self.threadList['relval'] = self.runPyRelVals(self.getDepThreads(['addon']))
if not only or 'libcheck' in only:
print '\n'+80*'-'+' libcheck\n'
self.threadList['libcheck'] = self.checkLibDeps()
if not only or 'geom' in only:
print '\n'+80*'-'+' geom \n'
self.threadList['geom'] = self.runGeomTests()
if not only or 'pyConfigs' in only:
print '\n'+80*'-'+' pyConfigs \n'
self.threadList['pyConfigs'] = self.checkPyConfigs()
if not only or 'dupDict' in only:
print '\n'+80*'-'+' dupDict \n'
self.threadList['dupDict'] = self.runDuplicateDictCheck()
print 'TestWait> waiting for tests to finish ....'
for task in self.threadList:
if self.threadList[task]: self.threadList[task].join()
print 'TestWait> Tests finished '
return
# --------------------------------------------------------------------------------
def checkPyConfigs(self, deps = []):
print "Going to check python configs in ", os.getcwd()
cmd = 'cd src ; '+scriptPath+'/checkPyConfigs.py > ../chkPyConf.log 2>&1'
try:
self.doCmd(cmd,self.dryRun,self.cmsswBuildDir)
self.logger.updateLogFile("chkPyConf.log")
self.logger.updateLogFile("chkPyConf.log",'testLogs')
except:
pass
return None
# --------------------------------------------------------------------------------
def checkLibDeps(self, deps = []):
print "libDepTests> Going to run LibDepChk ... "
thrd = None
try:
thrd = LibDepsTester(self.cmsswBuildDir,self.logger, deps)
thrd.start()
except Exception, e :
print "ERROR during LibDepChk : caught exception: " + str(e)
return thrd
class ReleaseTester(BuilderBase):
def __init__(self, buildDir, releaseDir, appset, doInstall=True):
BuilderBase.__init__(self)
os.environ['STAGE_SVCCLASS'] = 't1transfer'
os.environ['STAGE_HOST'] = 'castorcms'
os.environ['STAGER_TRACE'] = '3'
self.doInstall = doInstall
self.buildDir = buildDir
self.appset = appset
self.updateTimeStamp(self.buildDir)
self.cmsswBuildDir = releaseDir
self.release = os.path.basename(releaseDir)
self.relTag = self.release
self.relCycle = self.release.replace("CMSSW_",
"").split("_X_",
1)[0].replace("_", ".")
day, hour = self.stamp.split('-')
self.threadList = {}
self.maxThreads = 2 # one for unit-tests, one for addOnTests
self.doRelVal = True
self.logger = None
self.RelValArgs = None
config.setDefaults(self.relCycle)
try:
self.doRelVal = config.Configuration[self.relCycle]['runRelVal']
self.RelValArgs = config.Configuration[self.relCycle]['RelValArgs']
except:
pass
return
# --------------------------------------------------------------------------------
def setupLogger(self, doInstall=True):
if not self.logger:
from logUpdater import LogUpdater
self.logger = LogUpdater(self.buildDir, doInstall)
self.logger.setDryRun(self.dryRun)
self.logger.setRelease(self.relTag, self.relCycle, self.stamp)
return
# --------------------------------------------------------------------------------
def getDepThreads(self, jobs=[]):
deps = []
for job in jobs:
if self.threadList.has_key(job) and self.threadList[job]:
deps.append(self.threadList[job])
return deps
# --------------------------------------------------------------------------------
def doTest(self, only=None):
if not self.release:
print "ReleaseTester> ERROR: no release specified !! "
return
self.setupLogger(self.doInstall)
self.runProjectInit()
if not only or 'dirsize' in only:
print '\n' + 80 * '-' + ' dirsize \n'
self.threadList['dirsize'] = self.runDirSize()
if not only or 'depViolation' in only:
print '\n' + 80 * '-' + ' depViolation \n'
self.threadList['ddepViolation'] = self.runBuildFileDeps()
if not only or 'relProducts' in only:
print '\n' + 80 * '-' + ' relProducts \n'
self.threadList['relProducts'] = self.runReleaseProducts()
if not only or 'addon' in only:
print '\n' + 80 * '-' + ' addOnTests \n'
self.threadList['addon'] = self.runAddOnTests()
if not only or 'relvalcomp' in only:
print '\n' + 80 * '-' + ' relvalcomp \n'
self.threadList['relvalcomp'] = self.runBuildReports()
if not only or 'unit' in only:
print '\n' + 80 * '-' + ' unit \n'
self.threadList['unit'] = self.runUnitTests()
if not only or 'codeRules' in only:
print '\n' + 80 * '-' + ' codeRules \n'
self.threadList['codeRules'] = self.runCodeRulesChecker()
if not only or 'ignominy' in only:
print '\n' + 80 * '-' + ' ignominy \n'
self.threadList['ignominy'] = self.runIgnominy()
if not only or 'fwbuildset' in only:
print '\n' + 80 * '-' + ' FWLite BuildSet\n'
self.threadList['fwbuildset'] = self.runFWLiteBuildSet(
self.getDepThreads(['ignominy']))
#if not only or 'onlbuildset' in only:
# print '\n'+80*'-'+' Online BuildSet\n'
# self.threadList['onlbuildset'] = self.runOnlineBuildSet(self.getDepThreads(['ignominy']))
if not only or 'dqmoffline' in only:
print '\n' + 80 * '-' + ' DQMOfflineTest \n'
self.threadList['dqmoffline'] = self.runDQMOfflineTests(
self.getDepThreads(['unit']))
if not only or 'relval' == only:
print '\n' + 80 * '-' + ' relval \n'
self.threadList['relval'] = self.runPyRelVals(
self.getDepThreads(['addon']))
if not only or 'libcheck' in only:
print '\n' + 80 * '-' + ' libcheck\n'
self.threadList['libcheck'] = self.checkLibDeps()
if not only or 'geom' in only:
print '\n' + 80 * '-' + ' geom \n'
self.threadList['geom'] = self.runGeomTests()
if not only or 'pyConfigs' in only:
print '\n' + 80 * '-' + ' pyConfigs \n'
self.threadList['pyConfigs'] = self.checkPyConfigs()
if not only or 'dupDict' in only:
print '\n' + 80 * '-' + ' dupDict \n'
self.threadList['dupDict'] = self.runDuplicateDictCheck()
print 'TestWait> waiting for tests to finish ....'
for task in self.threadList:
if self.threadList[task]: self.threadList[task].join()
print 'TestWait> Tests finished '
return
# --------------------------------------------------------------------------------
def checkPyConfigs(self, deps=[]):
print "Going to check python configs in ", os.getcwd()
cmd = 'cd src ; ' + scriptPath + '/checkPyConfigs.py > ../chkPyConf.log 2>&1'
try:
self.doCmd(cmd, self.dryRun, self.cmsswBuildDir)
self.logger.updateLogFile("chkPyConf.log")
self.logger.updateLogFile("chkPyConf.log", 'testLogs')
except:
pass
return None
# --------------------------------------------------------------------------------
def checkLibDeps(self, deps=[]):
print "libDepTests> Going to run LibDepChk ... "
thrd = None
try:
thrd = LibDepsTester(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception, e:
print "ERROR during LibDepChk : caught exception: " + str(e)
return thrd
