class ReleaseTester(object):
def __init__(self, releaseDir, dryRun=False):
self.dryRun = dryRun
self.plat = os.environ["SCRAM_ARCH"]
self.appset = releaseDir + "/CMSDIST"
self.cmsswBuildDir = releaseDir
self.release = os.path.basename(releaseDir)
self.relTag = self.release
self.threadList = {}
from cmsutils import getIBReleaseInfo
self.relCycle, day, hour = getIBReleaseInfo(self.release)
from logUpdater import LogUpdater
self.logger = LogUpdater(self.cmsswBuildDir, self.dryRun)
return
# --------------------------------------------------------------------------------
def getDepThreads(self, jobs=[]):
deps = []
for job in jobs:
if job in self.threadList and self.threadList[job]:
deps.append(self.threadList[job])
return deps
# --------------------------------------------------------------------------------
def doTest(self, only=None):
if not self.release:
print("ReleaseTester> ERROR: no release specified !! ")
return
self.runProjectInit()
if not only or 'dirsize' in only:
print('\n' + 80 * '-' + ' dirsize \n')
self.threadList['dirsize'] = self.runDirSize()
if not only or 'depViolation' in only:
print('\n' + 80 * '-' + ' depViolation \n')
self.threadList['depViolation'] = self.runBuildFileDeps()
if not only or 'relProducts' in only:
print('\n' + 80 * '-' + ' relProducts \n')
self.threadList['relProducts'] = self.runReleaseProducts()
if not only or 'unit' in only:
print('\n' + 80 * '-' + ' unit \n')
self.threadList['unit'] = self.runUnitTests()
# We only want to explicitly run this test.
if only and 'gpu_unit' in only:
print('\n' + 80 * '-' + ' gpu_unit \n')
self.threadList['gpu_unit'] = self.runUnitTests([], 'GPU')
if not only or 'codeRules' in only:
print('\n' + 80 * '-' + ' codeRules \n')
self.threadList['codeRules'] = self.runCodeRulesChecker()
if not only or 'ignominy' in only:
print('\n' + 80 * '-' + ' ignominy \n')
self.threadList['ignominy'] = self.runIgnominy()
if not only or 'fwbuildset' in only:
print('\n' + 80 * '-' + ' FWLite BuildSet\n')
self.threadList['fwbuildset'] = self.runFWLiteBuildSet(
self.getDepThreads(['ignominy']))
if not only or 'libcheck' in only:
print('\n' + 80 * '-' + ' libcheck\n')
self.threadList['libcheck'] = self.checkLibDeps()
if not only or 'pyConfigs' in only:
print('\n' + 80 * '-' + ' pyConfigs \n')
self.threadList['pyConfigs'] = self.checkPyConfigs()
if not only or 'dupDict' in only:
print('\n' + 80 * '-' + ' dupDict \n')
self.threadList['dupDict'] = self.runDuplicateDictCheck()
print('TestWait> waiting for tests to finish ....')
for task in self.threadList:
if self.threadList[task]: self.threadList[task].join()
print('TestWait> Tests finished ')
return
# --------------------------------------------------------------------------------
def checkPyConfigs(self, deps=[]):
print("Going to check python configs in ", os.getcwd())
cmd = scriptPath + '/checkPyConfigs.py > chkPyConf.log 2>&1'
try:
doCmd(cmd, self.dryRun, self.cmsswBuildDir)
self.logger.updateLogFile("chkPyConf.log")
self.logger.updateLogFile("chkPyConf.log", 'testLogs')
except:
pass
return None
# --------------------------------------------------------------------------------
def checkLibDeps(self, deps=[]):
print("libDepTests> Going to run LibDepChk ... ")
thrd = None
try:
thrd = LibDepsTester(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during LibDepChk : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runProjectInit(self, deps=[]):
print("runProjectInit> Going regenerate scram caches ... ")
try:
ver = os.environ["CMSSW_VERSION"]
cmd = "cd " + self.cmsswBuildDir + "; rm -rf src;"
cmd += "curl -k -L -s -o src.tar.gz https://github.com/cms-sw/cmssw/archive/" + ver + ".tar.gz;"
cmd += "tar -xzf src.tar.gz; mv cmssw-" + ver + " src; rm -rf src.tar.gz;"
cmd += "mv src/Geometry/TrackerSimData/data src/Geometry/TrackerSimData/data.backup;"
cmd += "scram build -r echo_CXX"
doCmd(cmd)
except Exception as e:
print("ERROR during runProjectInit: caught exception: " + str(e))
return None
# --------------------------------------------------------------------------------
def runCodeRulesChecker(self, deps=[]):
print("runCodeRulesTests> Going to run cmsCodeRulesChecker ... ")
thrd = None
try:
thrd = CodeRulesChecker(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during cmsCodeRulesChecker : caught exception: " +
str(e))
return thrd
# --------------------------------------------------------------------------------
def runDuplicateDictCheck(self, deps=[]):
print(
"runDuplicateDictTests> Going to run duplicateReflexLibrarySearch.py ... "
)
script = 'duplicateReflexLibrarySearch.py'
for opt in ['dup', 'lostDefs', 'edmPD']:
cmd = script + ' --' + opt + ' 2>&1 >dupDict-' + opt + '.log'
try:
doCmd(cmd, self.dryRun, self.cmsswBuildDir)
except Exception as e:
print(
"ERROR during test duplicateDictCheck : caught exception: "
+ str(e))
self.logger.updateDupDictTestLogs()
return None
# --------------------------------------------------------------------------------
def runIgnominy(self, deps=[]):
print("ignominyTests> Going to run ignominy tests ... ")
thrd = None
try:
thrd = IgnominyTests(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during run ignominytests : caught exception: " +
str(e))
return thrd
# --------------------------------------------------------------------------------
def runFWLiteBuildSet(self, deps=[]):
print("FWLiteBuildSet> Going to run FWLite BuildSet tests ... ")
thd = None
try:
thd = AppBuildSetTests(self.cmsswBuildDir, self.logger,
self.appset, deps, 'fwlite')
thd.start()
except Exception as e:
print("ERROR during run FWLiteBuildSet : caught exception: " +
str(e))
return thd
# --------------------------------------------------------------------------------
def runUnitTests(self, deps=[], xType=""):
print("runTests> Going to run units tests ... ")
thrd = None
try:
thrd = UnitTester(self.cmsswBuildDir, self.logger, deps, xType)
thrd.start()
except Exception as e:
print("ERROR during run unittests : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runDirSize(self, deps=[]):
print("runTests> Going to run DirSize ... ")
thrd = None
try:
thrd = DirSizeTester(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during DirSize : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runReleaseProducts(self, deps=[]):
print("runTests> Going to run ReleaseProducts ... ")
thrd = None
try:
thrd = ReleaseProductsDump(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during ReleaseProducts : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runBuildFileDeps(self, deps=[]):
print("runTests> Going to run BuildFileDeps ... ")
thrd = None
try:
thrd = BuildFileDependencyCheck(self.cmsswBuildDir, self.logger,
deps)
thrd.start()
except Exception as e:
print("ERROR during RBuildFileDeps : caught exception: " + str(e))
return thrd
class ReleaseTester(object):
def __init__(self, releaseDir, dryRun=False):
self.dryRun = dryRun
self.plat = os.environ["SCRAM_ARCH"]
self.appset = releaseDir + "/CMSDIST"
self.cmsswBuildDir = releaseDir
self.release = os.path.basename(releaseDir)
self.relTag = self.release
self.threadList = {}
from cmsutils import getIBReleaseInfo
self.relCycle, day, hour = getIBReleaseInfo(self.release)
from logUpdater import LogUpdater
self.logger = LogUpdater(self.cmsswBuildDir, self.dryRun)
return
# --------------------------------------------------------------------------------
def getDepThreads(self, jobs=[]):
deps = []
for job in jobs:
if job in self.threadList and self.threadList[job]: deps.append(self.threadList[job])
return deps
# --------------------------------------------------------------------------------
def doTest(self, only=None):
if not self.release:
print("ReleaseTester> ERROR: no release specified !! ")
return
self.runProjectInit()
if not only or 'dirsize' in only:
print('\n' + 80 * '-' + ' dirsize \n')
self.threadList['dirsize'] = self.runDirSize()
if not only or 'depViolation' in only:
print('\n' + 80 * '-' + ' depViolation \n')
self.threadList['depViolation'] = self.runBuildFileDeps()
if not only or 'relProducts' in only:
print('\n' + 80 * '-' + ' relProducts \n')
self.threadList['relProducts'] = self.runReleaseProducts()
if not only or 'unit' in only:
print('\n' + 80 * '-' + ' unit \n')
self.threadList['unit'] = self.runUnitTests()
# We only want to explicitly run this test.
if only and 'gpu_unit' in only:
print('\n' + 80 * '-' + ' gpu_unit \n')
self.threadList['gpu_unit'] = self.runUnitTests([], 'GPU')
if not only or 'codeRules' in only:
print('\n' + 80 * '-' + ' codeRules \n')
self.threadList['codeRules'] = self.runCodeRulesChecker()
if not only or 'ignominy' in only:
print('\n' + 80 * '-' + ' ignominy \n')
self.threadList['ignominy'] = self.runIgnominy()
if not only or 'fwbuildset' in only:
print('\n' + 80 * '-' + ' FWLite BuildSet\n')
self.threadList['fwbuildset'] = self.runFWLiteBuildSet(self.getDepThreads(['ignominy']))
if not only or 'libcheck' in only:
print('\n' + 80 * '-' + ' libcheck\n')
self.threadList['libcheck'] = self.checkLibDeps()
if not only or 'pyConfigs' in only:
print('\n' + 80 * '-' + ' pyConfigs \n')
self.threadList['pyConfigs'] = self.checkPyConfigs()
if not only or 'dupDict' in only:
print('\n' + 80 * '-' + ' dupDict \n')
self.threadList['dupDict'] = self.runDuplicateDictCheck()
print('TestWait> waiting for tests to finish ....')
for task in self.threadList:
if self.threadList[task]: self.threadList[task].join()
print('TestWait> Tests finished ')
return
# --------------------------------------------------------------------------------
def checkPyConfigs(self, deps=[]):
print("Going to check python configs in ", os.getcwd())
cmd = scriptPath + '/checkPyConfigs.py > chkPyConf.log 2>&1'
try:
doCmd(cmd, self.dryRun, self.cmsswBuildDir)
self.logger.updateLogFile("chkPyConf.log")
self.logger.updateLogFile("chkPyConf.log", 'testLogs')
except:
pass
return None
# --------------------------------------------------------------------------------
def checkLibDeps(self, deps=[]):
print("libDepTests> Going to run LibDepChk ... ")
thrd = None
try:
thrd = LibDepsTester(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during LibDepChk : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runProjectInit(self, deps=[]):
print("runProjectInit> Going regenerate scram caches ... ")
try:
ver = os.environ["CMSSW_VERSION"]
cmd = "cd " + self.cmsswBuildDir + "; rm -rf src;"
cmd += "curl -k -L -s -o src.tar.gz https://github.com/cms-sw/cmssw/archive/" + ver + ".tar.gz;"
cmd += "tar -xzf src.tar.gz; mv cmssw-" + ver + " src; rm -rf src.tar.gz;"
cmd += "mv src/Geometry/TrackerSimData/data src/Geometry/TrackerSimData/data.backup;"
cmd += "scram build -r echo_CXX"
doCmd(cmd)
except Exception as e:
print("ERROR during runProjectInit: caught exception: " + str(e))
return None
# --------------------------------------------------------------------------------
def runCodeRulesChecker(self, deps=[]):
print("runCodeRulesTests> Going to run cmsCodeRulesChecker ... ")
thrd = None
try:
thrd = CodeRulesChecker(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during cmsCodeRulesChecker : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runDuplicateDictCheck(self, deps=[]):
print("runDuplicateDictTests> Going to run duplicateReflexLibrarySearch.py ... ")
script = 'duplicateReflexLibrarySearch.py'
for opt in ['dup', 'lostDefs', 'edmPD']:
cmd = script + ' --' + opt + ' 2>&1 >dupDict-' + opt + '.log'
try:
doCmd(cmd, self.dryRun, self.cmsswBuildDir)
except Exception as e:
print("ERROR during test duplicateDictCheck : caught exception: " + str(e))
self.logger.updateDupDictTestLogs()
return None
# --------------------------------------------------------------------------------
def runIgnominy(self, deps=[]):
print("ignominyTests> Going to run ignominy tests ... ")
thrd = None
try:
thrd = IgnominyTests(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during run ignominytests : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runFWLiteBuildSet(self, deps=[]):
print("FWLiteBuildSet> Going to run FWLite BuildSet tests ... ")
thd = None
try:
thd = AppBuildSetTests(self.cmsswBuildDir, self.logger, self.appset, deps, 'fwlite')
thd.start()
except Exception as e:
print("ERROR during run FWLiteBuildSet : caught exception: " + str(e))
return thd
# --------------------------------------------------------------------------------
def runUnitTests(self, deps=[], xType=""):
print("runTests> Going to run units tests ... ")
thrd = None
try:
thrd = UnitTester(self.cmsswBuildDir, self.logger, deps, xType)
thrd.start()
except Exception as e:
print("ERROR during run unittests : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runDirSize(self, deps=[]):
print("runTests> Going to run DirSize ... ")
thrd = None
try:
thrd = DirSizeTester(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during DirSize : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runReleaseProducts(self, deps=[]):
print("runTests> Going to run ReleaseProducts ... ")
thrd = None
try:
thrd = ReleaseProductsDump(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during ReleaseProducts : caught exception: " + str(e))
return thrd
# --------------------------------------------------------------------------------
def runBuildFileDeps(self, deps=[]):
print("runTests> Going to run BuildFileDeps ... ")
thrd = None
try:
thrd = BuildFileDependencyCheck(self.cmsswBuildDir, self.logger, deps)
thrd.start()
except Exception as e:
print("ERROR during RBuildFileDeps : caught exception: " + str(e))
return thrd
