class ReleaseTester(object): def __init__(self, releaseDir, dryRun=False): self.dryRun = dryRun self.plat = os.environ["SCRAM_ARCH"] self.appset = releaseDir + "/CMSDIST" self.cmsswBuildDir = releaseDir self.release = os.path.basename(releaseDir) self.relTag = self.release self.threadList = {} from cmsutils import getIBReleaseInfo self.relCycle, day, hour = getIBReleaseInfo(self.release) from logUpdater import LogUpdater self.logger = LogUpdater(self.cmsswBuildDir, self.dryRun) return # -------------------------------------------------------------------------------- def getDepThreads(self, jobs=[]): deps = [] for job in jobs: if job in self.threadList and self.threadList[job]: deps.append(self.threadList[job]) return deps # -------------------------------------------------------------------------------- def doTest(self, only=None): if not self.release: print("ReleaseTester> ERROR: no release specified !! ") return self.runProjectInit() if not only or 'dirsize' in only: print('\n' + 80 * '-' + ' dirsize \n') self.threadList['dirsize'] = self.runDirSize() if not only or 'depViolation' in only: print('\n' + 80 * '-' + ' depViolation \n') self.threadList['depViolation'] = self.runBuildFileDeps() if not only or 'relProducts' in only: print('\n' + 80 * '-' + ' relProducts \n') self.threadList['relProducts'] = self.runReleaseProducts() if not only or 'unit' in only: print('\n' + 80 * '-' + ' unit \n') self.threadList['unit'] = self.runUnitTests() # We only want to explicitly run this test. if only and 'gpu_unit' in only: print('\n' + 80 * '-' + ' gpu_unit \n') self.threadList['gpu_unit'] = self.runUnitTests([], 'GPU') if not only or 'codeRules' in only: print('\n' + 80 * '-' + ' codeRules \n') self.threadList['codeRules'] = self.runCodeRulesChecker() if not only or 'ignominy' in only: print('\n' + 80 * '-' + ' ignominy \n') self.threadList['ignominy'] = self.runIgnominy() if not only or 'fwbuildset' in only: print('\n' + 80 * '-' + ' FWLite BuildSet\n') self.threadList['fwbuildset'] = self.runFWLiteBuildSet( self.getDepThreads(['ignominy'])) if not only or 'libcheck' in only: print('\n' + 80 * '-' + ' libcheck\n') self.threadList['libcheck'] = self.checkLibDeps() if not only or 'pyConfigs' in only: print('\n' + 80 * '-' + ' pyConfigs \n') self.threadList['pyConfigs'] = self.checkPyConfigs() if not only or 'dupDict' in only: print('\n' + 80 * '-' + ' dupDict \n') self.threadList['dupDict'] = self.runDuplicateDictCheck() print('TestWait> waiting for tests to finish ....') for task in self.threadList: if self.threadList[task]: self.threadList[task].join() print('TestWait> Tests finished ') return # -------------------------------------------------------------------------------- def checkPyConfigs(self, deps=[]): print("Going to check python configs in ", os.getcwd()) cmd = scriptPath + '/checkPyConfigs.py > chkPyConf.log 2>&1' try: doCmd(cmd, self.dryRun, self.cmsswBuildDir) self.logger.updateLogFile("chkPyConf.log") self.logger.updateLogFile("chkPyConf.log", 'testLogs') except: pass return None # -------------------------------------------------------------------------------- def checkLibDeps(self, deps=[]): print("libDepTests> Going to run LibDepChk ... ") thrd = None try: thrd = LibDepsTester(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during LibDepChk : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runProjectInit(self, deps=[]): print("runProjectInit> Going regenerate scram caches ... ") try: ver = os.environ["CMSSW_VERSION"] cmd = "cd " + self.cmsswBuildDir + "; rm -rf src;" cmd += "curl -k -L -s -o src.tar.gz https://github.com/cms-sw/cmssw/archive/" + ver + ".tar.gz;" cmd += "tar -xzf src.tar.gz; mv cmssw-" + ver + " src; rm -rf src.tar.gz;" cmd += "mv src/Geometry/TrackerSimData/data src/Geometry/TrackerSimData/data.backup;" cmd += "scram build -r echo_CXX" doCmd(cmd) except Exception as e: print("ERROR during runProjectInit: caught exception: " + str(e)) return None # -------------------------------------------------------------------------------- def runCodeRulesChecker(self, deps=[]): print("runCodeRulesTests> Going to run cmsCodeRulesChecker ... ") thrd = None try: thrd = CodeRulesChecker(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during cmsCodeRulesChecker : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runDuplicateDictCheck(self, deps=[]): print( "runDuplicateDictTests> Going to run duplicateReflexLibrarySearch.py ... " ) script = 'duplicateReflexLibrarySearch.py' for opt in ['dup', 'lostDefs', 'edmPD']: cmd = script + ' --' + opt + ' 2>&1 >dupDict-' + opt + '.log' try: doCmd(cmd, self.dryRun, self.cmsswBuildDir) except Exception as e: print( "ERROR during test duplicateDictCheck : caught exception: " + str(e)) self.logger.updateDupDictTestLogs() return None # -------------------------------------------------------------------------------- def runIgnominy(self, deps=[]): print("ignominyTests> Going to run ignominy tests ... ") thrd = None try: thrd = IgnominyTests(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during run ignominytests : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runFWLiteBuildSet(self, deps=[]): print("FWLiteBuildSet> Going to run FWLite BuildSet tests ... ") thd = None try: thd = AppBuildSetTests(self.cmsswBuildDir, self.logger, self.appset, deps, 'fwlite') thd.start() except Exception as e: print("ERROR during run FWLiteBuildSet : caught exception: " + str(e)) return thd # -------------------------------------------------------------------------------- def runUnitTests(self, deps=[], xType=""): print("runTests> Going to run units tests ... ") thrd = None try: thrd = UnitTester(self.cmsswBuildDir, self.logger, deps, xType) thrd.start() except Exception as e: print("ERROR during run unittests : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runDirSize(self, deps=[]): print("runTests> Going to run DirSize ... ") thrd = None try: thrd = DirSizeTester(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during DirSize : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runReleaseProducts(self, deps=[]): print("runTests> Going to run ReleaseProducts ... ") thrd = None try: thrd = ReleaseProductsDump(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during ReleaseProducts : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runBuildFileDeps(self, deps=[]): print("runTests> Going to run BuildFileDeps ... ") thrd = None try: thrd = BuildFileDependencyCheck(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during RBuildFileDeps : caught exception: " + str(e)) return thrd
class ReleaseTester(object): def __init__(self, releaseDir, dryRun=False): self.dryRun = dryRun self.plat = os.environ["SCRAM_ARCH"] self.appset = releaseDir + "/CMSDIST" self.cmsswBuildDir = releaseDir self.release = os.path.basename(releaseDir) self.relTag = self.release self.threadList = {} from cmsutils import getIBReleaseInfo self.relCycle, day, hour = getIBReleaseInfo(self.release) from logUpdater import LogUpdater self.logger = LogUpdater(self.cmsswBuildDir, self.dryRun) return # -------------------------------------------------------------------------------- def getDepThreads(self, jobs=[]): deps = [] for job in jobs: if job in self.threadList and self.threadList[job]: deps.append(self.threadList[job]) return deps # -------------------------------------------------------------------------------- def doTest(self, only=None): if not self.release: print("ReleaseTester> ERROR: no release specified !! ") return self.runProjectInit() if not only or 'dirsize' in only: print('\n' + 80 * '-' + ' dirsize \n') self.threadList['dirsize'] = self.runDirSize() if not only or 'depViolation' in only: print('\n' + 80 * '-' + ' depViolation \n') self.threadList['depViolation'] = self.runBuildFileDeps() if not only or 'relProducts' in only: print('\n' + 80 * '-' + ' relProducts \n') self.threadList['relProducts'] = self.runReleaseProducts() if not only or 'unit' in only: print('\n' + 80 * '-' + ' unit \n') self.threadList['unit'] = self.runUnitTests() # We only want to explicitly run this test. if only and 'gpu_unit' in only: print('\n' + 80 * '-' + ' gpu_unit \n') self.threadList['gpu_unit'] = self.runUnitTests([], 'GPU') if not only or 'codeRules' in only: print('\n' + 80 * '-' + ' codeRules \n') self.threadList['codeRules'] = self.runCodeRulesChecker() if not only or 'ignominy' in only: print('\n' + 80 * '-' + ' ignominy \n') self.threadList['ignominy'] = self.runIgnominy() if not only or 'fwbuildset' in only: print('\n' + 80 * '-' + ' FWLite BuildSet\n') self.threadList['fwbuildset'] = self.runFWLiteBuildSet(self.getDepThreads(['ignominy'])) if not only or 'libcheck' in only: print('\n' + 80 * '-' + ' libcheck\n') self.threadList['libcheck'] = self.checkLibDeps() if not only or 'pyConfigs' in only: print('\n' + 80 * '-' + ' pyConfigs \n') self.threadList['pyConfigs'] = self.checkPyConfigs() if not only or 'dupDict' in only: print('\n' + 80 * '-' + ' dupDict \n') self.threadList['dupDict'] = self.runDuplicateDictCheck() print('TestWait> waiting for tests to finish ....') for task in self.threadList: if self.threadList[task]: self.threadList[task].join() print('TestWait> Tests finished ') return # -------------------------------------------------------------------------------- def checkPyConfigs(self, deps=[]): print("Going to check python configs in ", os.getcwd()) cmd = scriptPath + '/checkPyConfigs.py > chkPyConf.log 2>&1' try: doCmd(cmd, self.dryRun, self.cmsswBuildDir) self.logger.updateLogFile("chkPyConf.log") self.logger.updateLogFile("chkPyConf.log", 'testLogs') except: pass return None # -------------------------------------------------------------------------------- def checkLibDeps(self, deps=[]): print("libDepTests> Going to run LibDepChk ... ") thrd = None try: thrd = LibDepsTester(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during LibDepChk : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runProjectInit(self, deps=[]): print("runProjectInit> Going regenerate scram caches ... ") try: ver = os.environ["CMSSW_VERSION"] cmd = "cd " + self.cmsswBuildDir + "; rm -rf src;" cmd += "curl -k -L -s -o src.tar.gz https://github.com/cms-sw/cmssw/archive/" + ver + ".tar.gz;" cmd += "tar -xzf src.tar.gz; mv cmssw-" + ver + " src; rm -rf src.tar.gz;" cmd += "mv src/Geometry/TrackerSimData/data src/Geometry/TrackerSimData/data.backup;" cmd += "scram build -r echo_CXX" doCmd(cmd) except Exception as e: print("ERROR during runProjectInit: caught exception: " + str(e)) return None # -------------------------------------------------------------------------------- def runCodeRulesChecker(self, deps=[]): print("runCodeRulesTests> Going to run cmsCodeRulesChecker ... ") thrd = None try: thrd = CodeRulesChecker(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during cmsCodeRulesChecker : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runDuplicateDictCheck(self, deps=[]): print("runDuplicateDictTests> Going to run duplicateReflexLibrarySearch.py ... ") script = 'duplicateReflexLibrarySearch.py' for opt in ['dup', 'lostDefs', 'edmPD']: cmd = script + ' --' + opt + ' 2>&1 >dupDict-' + opt + '.log' try: doCmd(cmd, self.dryRun, self.cmsswBuildDir) except Exception as e: print("ERROR during test duplicateDictCheck : caught exception: " + str(e)) self.logger.updateDupDictTestLogs() return None # -------------------------------------------------------------------------------- def runIgnominy(self, deps=[]): print("ignominyTests> Going to run ignominy tests ... ") thrd = None try: thrd = IgnominyTests(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during run ignominytests : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runFWLiteBuildSet(self, deps=[]): print("FWLiteBuildSet> Going to run FWLite BuildSet tests ... ") thd = None try: thd = AppBuildSetTests(self.cmsswBuildDir, self.logger, self.appset, deps, 'fwlite') thd.start() except Exception as e: print("ERROR during run FWLiteBuildSet : caught exception: " + str(e)) return thd # -------------------------------------------------------------------------------- def runUnitTests(self, deps=[], xType=""): print("runTests> Going to run units tests ... ") thrd = None try: thrd = UnitTester(self.cmsswBuildDir, self.logger, deps, xType) thrd.start() except Exception as e: print("ERROR during run unittests : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runDirSize(self, deps=[]): print("runTests> Going to run DirSize ... ") thrd = None try: thrd = DirSizeTester(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during DirSize : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runReleaseProducts(self, deps=[]): print("runTests> Going to run ReleaseProducts ... ") thrd = None try: thrd = ReleaseProductsDump(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during ReleaseProducts : caught exception: " + str(e)) return thrd # -------------------------------------------------------------------------------- def runBuildFileDeps(self, deps=[]): print("runTests> Going to run BuildFileDeps ... ") thrd = None try: thrd = BuildFileDependencyCheck(self.cmsswBuildDir, self.logger, deps) thrd.start() except Exception as e: print("ERROR during RBuildFileDeps : caught exception: " + str(e)) return thrd