def __init__(self, buildDir, releaseDir, appset, doInstall=True): BuilderBase.__init__(self) os.environ['STAGE_SVCCLASS']='t1transfer' os.environ['STAGE_HOST']='castorcms' os.environ['STAGER_TRACE']='3' self.doInstall=doInstall self.buildDir = buildDir self.appset = appset self.updateTimeStamp(self.buildDir) self.cmsswBuildDir = releaseDir self.release = os.path.basename(releaseDir) self.relTag = self.release self.relCycle = self.release.replace("CMSSW_","").split("_X_",1)[0].replace("_",".") day,hour = self.stamp.split('-') self.threadList = {} self.maxThreads = 2 # one for unit-tests, one for addOnTests self.doRelVal = True self.logger = None self.RelValArgs = None config.setDefaults(self.relCycle) try: self.doRelVal = config.Configuration[self.relCycle]['runRelVal'] self.RelValArgs = config.Configuration[self.relCycle]['RelValArgs'] except: pass return
def __init__(self, buildDir, releaseDir, appset, doInstall=True): BuilderBase.__init__(self) os.environ['STAGE_SVCCLASS'] = 't1transfer' os.environ['STAGE_HOST'] = 'castorcms' os.environ['STAGER_TRACE'] = '3' self.doInstall = doInstall self.buildDir = buildDir self.appset = appset self.updateTimeStamp(self.buildDir) self.cmsswBuildDir = releaseDir self.release = os.path.basename(releaseDir) self.relTag = self.release self.relCycle = self.release.replace("CMSSW_", "").split("_X_", 1)[0].replace("_", ".") day, hour = self.stamp.split('-') self.threadList = {} self.maxThreads = 2 # one for unit-tests, one for addOnTests self.doRelVal = True self.logger = None self.RelValArgs = None config.setDefaults(self.relCycle) try: self.doRelVal = config.Configuration[self.relCycle]['runRelVal'] self.RelValArgs = config.Configuration[self.relCycle]['RelValArgs'] except: pass return
def __init__(self, dayIn=None, relCycIn=None, dryRunIn=False,doInstall=True) : BuilderBase.__init__(self) self.day = dayIn if not self.day : self.day = time.strftime("%a").lower() self.dryRun = dryRunIn self.topDir = os.path.join(self.installDir, self.plat) self.relCycle = relCycIn self.doInstall = doInstall self.cand = None config.setDefaults (relCycIn) return
def main(): import getopt options = sys.argv[1:] try: opts, args = getopt.getopt( options, 'h', ['help', 'buildDir=', 'release=', 'tag=', 'dryRun', 'cycle=']) except getopt.GetoptError: usage() sys.exit(-2) buildDir = None rel = None tag = None dryRun = False cycle = None for o, a in opts: if o in ('-h', '--help'): usage() sys.exit() if o in ('--buildDir', ): buildDir = a if o in ('--release', ): rel = a if o in ('--tag', ): tag = a if o in ('--dryRun', ): dryRun = True if o in ('--cycle', ): cycle = a config.setDefaults(cycle) cfg = config.Configuration[cycle] rb = ReleaseBuilder(buildDir, cfg) if dryRun: rb.setDryRun() try: rb.doBuild(rel, tag) except Exception, e: print "ERROR: Caught exception during doBuild : " + str(e)
def main(): import getopt options = sys.argv[1:] try: opts, args = getopt.getopt(options, "h", ["help", "buildDir=", "release=", "tag=", "dryRun", "cycle="]) except getopt.GetoptError: usage() sys.exit(-2) buildDir = None rel = None tag = None dryRun = False cycle = None for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() if o in ("--buildDir",): buildDir = a if o in ("--release",): rel = a if o in ("--tag",): tag = a if o in ("--dryRun",): dryRun = True if o in ("--cycle",): cycle = a config.setDefaults(cycle) cfg = config.Configuration[cycle] rb = ReleaseBuilder(buildDir, cfg) if dryRun: rb.setDryRun() try: rb.doBuild(rel, tag) except Exception, e: print "ERROR: Caught exception during doBuild : " + str(e)
def __init__(self, dayIn=None, relCycIn=None, dryRunIn=False, doInstall=True): BuilderBase.__init__(self) self.day = dayIn if not self.day: self.day = time.strftime("%a").lower() self.dryRun = dryRunIn self.topDir = os.path.join(self.installDir, self.plat) self.relCycle = relCycIn self.doInstall = doInstall self.cand = None config.setDefaults(relCycIn) return
def main(): parser = OptionParser() usage = "usage: %prog [option1] arg1 [option2] arg2 ... [optionN] argN" parser = OptionParser(usage=usage) parser.add_option("-b", "--buildDir", dest="buildDir", help="build directory") parser.add_option("-r", "--releaseDir", dest="releaseDir", help="release directory") parser.add_option("-d", "--dryRun", action="store_true", dest="dryRun", help="is a dry run?") parser.add_option("-p", "--platform", dest="platform", help="used architecture") parser.add_option("-c", "--relCycle", dest="relCycle", help="Release cycle") parser.add_option("-s", "--stamp", dest="stamp", help="Release stamp") parser.add_option("-t", "--relTag", dest="relTag", help="Release tag") parser.add_option("-u", "--custom_ib", action="store_true", dest="custom_ib", help="Sets the specific configuration for running DQM comparison for Custom IBs.") parser.add_option("-e", "--report_path", dest="report_path", help="Path to report. Optional value") parser.add_option("-o", "--report_relative_path", dest="report_relative_path", help="Relative Path to report. The value is used for compiling report web page url. Optional value") (options, args) = parser.parse_args() dryRun = False if not options.dryRun == None: dryRun = options.dryRun custom_ib = False if not options.custom_ib == None: custom_ib = options.custom_ib if options.buildDir == None: print "The Integration Build directory is not specified" return errno.EINVAL buildDir = options.buildDir report_path = options.report_path report_relative_path = options.report_relative_path while buildDir.endswith('/'): buildDir = buildDir[:(len(buildDir)-1)] (head, stamp) = os.path.split(buildDir) head.strip() #hack, don't show never read warning if options.stamp == None: stamp = str(stamp).strip() else: stamp = options.stamp if options.relTag == None: buildName = getBuildName(buildDir) else: buildName = options.relTag if options.platform == None: buildArchitecture = getBuildArchitecture(buildName, buildDir) else: buildArchitecture = options.platform if options.relCycle == None: rel = getRelVersion(buildName) else: rel = options.relCycle intBldFQNPath = os.path.dirname(os.path.dirname(buildDir)) if not options.releaseDir == None: buildNameFQNPath = options.releaseDir else: buildNameFQNPath = os.path.join(buildDir, buildName) if not os.path.exists(buildNameFQNPath): print 'Build is not found in common path %s' % str(buildNameFQNPath) buildNameFQNPath = os.path.join(intBldFQNPath, 'cms', buildArchitecture, 'cms', 'cmssw', buildName) print 'Looking for the build in %s' % str(buildNameFQNPath) if not os.path.exists(buildNameFQNPath): print 'The build location cannot be found, terminating ...' return errno.ENOENT import config config.setDefaults(rel) relMonParameters = config.Configuration[rel]['RelMonParameters'] if custom_ib: deployer_spec_file = os.path.join( buildNameFQNPath, 'tmp', 'tmpspec-cmssw-dqm-reference-deployer') referenceBuildName = config.getDQMReferenceBuild(deployer_spec_file) else: referenceBuildName = config.getDQMReferenceBuild() if not referenceBuildName: print "Reference build doesn't exist in configuration, exiting comparison ..." return errno.ENOENT referenceBuildFQNPath = getReferenceBuildFQNPath(config, rel, referenceBuildName, buildArchitecture); if not os.path.exists(referenceBuildFQNPath): referenceBuildFQNPath2 = os.path.join(buildNameFQNPath, buildArchitecture, 'cms', 'cmssw-dqm-reference-deployer', referenceBuildName, 'data') print "Reference build path %s doesn't exist, will search in %s" % (referenceBuildFQNPath,referenceBuildFQNPath2,) referenceBuildFQNPath = referenceBuildFQNPath2 (referenceBuildName, referenceBuildArchitecture) = searchReferenceBuildNameAndArchitectureInPath(referenceBuildFQNPath) threshold = getRelMonValue(relMonParameters,'threshold') statTest = getRelMonValue(relMonParameters,'statTest') doPngs = getRelMonValue(relMonParameters,'doPngs') doComparison = getRelMonValue(relMonParameters,'doComparison') doReport = getRelMonValue(relMonParameters,'doReport') no_successes = getRelMonValue(relMonParameters,'no_successes') black_list = getRelMonValue(relMonParameters,'black_list') success_percentage = getRelMonValue(relMonParameters, 'success_percentage') workflowFQNParentPath = os.path.join(buildNameFQNPath, workflowDataRelativePath) if not os.path.exists(workflowFQNParentPath): workflowFQNParentPath2 = os.path.join(buildNameFQNPath, workflowDataRelativePath_tests) print 'The workflow location %s cannot be found, will search in %s' % (workflowFQNParentPath, workflowFQNParentPath2,) workflowFQNParentPath = workflowFQNParentPath2 if not os.path.exists(workflowFQNParentPath): print 'The workflow location %s cannot be found, terminating ...' % workflowFQNParentPath return errno.ENOENT dirs = os.listdir(workflowFQNParentPath) wThreads = {} for workflowDir in dirs: if 'HARVEST' in workflowDir: workflow = getWorkflowNumber(workflowDir) workflowDirFQN = os.path.join(workflowFQNParentPath, workflowDir) dqmHarvestedRootFile = getDQMHarvestedRootFile(workflowDirFQN) if dqmHarvestedRootFile is not None: rootfileBuildFQN = os.path.join(workflowDirFQN, dqmHarvestedRootFile) ## reference build root file referenceWorkflows = os.listdir(referenceBuildFQNPath) referenceBuildWorkflowFQNPath = None for referenceWorkflow in referenceWorkflows: if referenceWorkflow.startswith(str(workflow)): referenceBuildWorkflowFQNPath = os.path.join(referenceBuildFQNPath, referenceWorkflow) break if referenceBuildWorkflowFQNPath == None: print "Workflow ", str(workflow), " doesn't exist in the reference build folder ", str(referenceBuildFQNPath) continue #go to next workflow iw current one is missing refRootfileBuild = getDQMHarvestedRootFile(referenceBuildWorkflowFQNPath) if refRootfileBuild is not None: refRootfileBuildFQN = os.path.join(referenceBuildWorkflowFQNPath, refRootfileBuild) print "comparing file ", rootfileBuildFQN, " with ", refRootfileBuildFQN outputReportName = workflow outputReportDir = os.path.join(buildNameFQNPath, outputReportBase, outputReportName) increment = 0 while os.path.exists(outputReportDir): increment = increment + 1 outputReportDir = outputReportDir + '-' + str(increment) os.makedirs(outputReportDir) waitThreads(wThreads,config.MachineCPUCount) thrd = Thread(target=makeComparison, args=(buildNameFQNPath, buildArchitecture, no_successes, rootfileBuildFQN, refRootfileBuildFQN, outputReportDir, threshold, statTest, doPngs, doComparison, doReport, workflowDir, workflowFQNParentPath, black_list, success_percentage, stamp, rel, outputReportName, buildName, workflow, referenceBuildName, report_path, report_relative_path, dryRun)) wThreads[thrd]=1 thrd.start() time.sleep(2) else: print '%s folder doesn\'t contain the referencing DQM harvested root file.' % workflowDir else: print '%s folder doesn\'t contain the DQM harvested root file.' % workflowDir waitThreads(wThreads) #if os.path.exists(outputReportDir): # print 'Deleting temporary build report dir output' # doCmd.doCmd('rm -rf %s' % outputReportDir, dryRun) print "Done" return
def main(): parser = OptionParser() usage = "usage: %prog [option1] arg1 [option2] arg2 ... [optionN] argN" parser = OptionParser(usage=usage) parser.add_option("-b", "--buildDir", dest="buildDir", help="build directory") parser.add_option("-r", "--releaseDir", dest="releaseDir", help="release directory") parser.add_option("-d", "--dryRun", action="store_true", dest="dryRun", help="is a dry run?") parser.add_option("-p", "--platform", dest="platform", help="used architecture") parser.add_option("-c", "--relCycle", dest="relCycle", help="Release cycle") parser.add_option("-s", "--stamp", dest="stamp", help="Release stamp") parser.add_option("-t", "--relTag", dest="relTag", help="Release tag") parser.add_option( "-u", "--custom_ib", action="store_true", dest="custom_ib", help= "Sets the specific configuration for running DQM comparison for Custom IBs." ) parser.add_option("-e", "--report_path", dest="report_path", help="Path to report. Optional value") parser.add_option( "-o", "--report_relative_path", dest="report_relative_path", help= "Relative Path to report. The value is used for compiling report web page url. Optional value" ) (options, args) = parser.parse_args() dryRun = False if not options.dryRun == None: dryRun = options.dryRun custom_ib = False if not options.custom_ib == None: custom_ib = options.custom_ib if options.buildDir == None: print "The Integration Build directory is not specified" return errno.EINVAL buildDir = options.buildDir report_path = options.report_path report_relative_path = options.report_relative_path while buildDir.endswith('/'): buildDir = buildDir[:(len(buildDir) - 1)] (head, stamp) = os.path.split(buildDir) head.strip() #hack, don't show never read warning if options.stamp == None: stamp = str(stamp).strip() else: stamp = options.stamp if options.relTag == None: buildName = getBuildName(buildDir) else: buildName = options.relTag if options.platform == None: buildArchitecture = getBuildArchitecture(buildName, buildDir) else: buildArchitecture = options.platform if options.relCycle == None: rel = getRelVersion(buildName) else: rel = options.relCycle intBldFQNPath = os.path.dirname(os.path.dirname(buildDir)) if not options.releaseDir == None: buildNameFQNPath = options.releaseDir else: buildNameFQNPath = os.path.join(buildDir, buildName) if not os.path.exists(buildNameFQNPath): print 'Build is not found in common path %s' % str(buildNameFQNPath) buildNameFQNPath = os.path.join(intBldFQNPath, 'cms', buildArchitecture, 'cms', 'cmssw', buildName) print 'Looking for the build in %s' % str(buildNameFQNPath) if not os.path.exists(buildNameFQNPath): print 'The build location cannot be found, terminating ...' return errno.ENOENT import config config.setDefaults(rel) relMonParameters = config.Configuration[rel]['RelMonParameters'] if custom_ib: deployer_spec_file = os.path.join( buildNameFQNPath, 'tmp', 'tmpspec-cmssw-dqm-reference-deployer') referenceBuildName = config.getDQMReferenceBuild(deployer_spec_file) else: referenceBuildName = config.getDQMReferenceBuild() if not referenceBuildName: print "Reference build doesn't exist in configuration, exiting comparison ..." return errno.ENOENT referenceBuildFQNPath = getReferenceBuildFQNPath(config, rel, referenceBuildName, buildArchitecture) if not os.path.exists(referenceBuildFQNPath): referenceBuildFQNPath2 = os.path.join(buildNameFQNPath, buildArchitecture, 'cms', 'cmssw-dqm-reference-deployer', referenceBuildName, 'data') print "Reference build path %s doesn't exist, will search in %s" % ( referenceBuildFQNPath, referenceBuildFQNPath2, ) referenceBuildFQNPath = referenceBuildFQNPath2 (referenceBuildName, referenceBuildArchitecture ) = searchReferenceBuildNameAndArchitectureInPath(referenceBuildFQNPath) threshold = getRelMonValue(relMonParameters, 'threshold') statTest = getRelMonValue(relMonParameters, 'statTest') doPngs = getRelMonValue(relMonParameters, 'doPngs') doComparison = getRelMonValue(relMonParameters, 'doComparison') doReport = getRelMonValue(relMonParameters, 'doReport') no_successes = getRelMonValue(relMonParameters, 'no_successes') black_list = getRelMonValue(relMonParameters, 'black_list') success_percentage = getRelMonValue(relMonParameters, 'success_percentage') workflowFQNParentPath = os.path.join(buildNameFQNPath, workflowDataRelativePath) if not os.path.exists(workflowFQNParentPath): workflowFQNParentPath2 = os.path.join(buildNameFQNPath, workflowDataRelativePath_tests) print 'The workflow location %s cannot be found, will search in %s' % ( workflowFQNParentPath, workflowFQNParentPath2, ) workflowFQNParentPath = workflowFQNParentPath2 if not os.path.exists(workflowFQNParentPath): print 'The workflow location %s cannot be found, terminating ...' % workflowFQNParentPath return errno.ENOENT dirs = os.listdir(workflowFQNParentPath) wThreads = {} for workflowDir in dirs: if 'HARVEST' in workflowDir: workflow = getWorkflowNumber(workflowDir) workflowDirFQN = os.path.join(workflowFQNParentPath, workflowDir) dqmHarvestedRootFile = getDQMHarvestedRootFile(workflowDirFQN) if dqmHarvestedRootFile is not None: rootfileBuildFQN = os.path.join(workflowDirFQN, dqmHarvestedRootFile) ## reference build root file referenceWorkflows = os.listdir(referenceBuildFQNPath) referenceBuildWorkflowFQNPath = None for referenceWorkflow in referenceWorkflows: if referenceWorkflow.startswith(str(workflow)): referenceBuildWorkflowFQNPath = os.path.join( referenceBuildFQNPath, referenceWorkflow) break if referenceBuildWorkflowFQNPath == None: print "Workflow ", str( workflow ), " doesn't exist in the reference build folder ", str( referenceBuildFQNPath) continue #go to next workflow iw current one is missing refRootfileBuild = getDQMHarvestedRootFile( referenceBuildWorkflowFQNPath) if refRootfileBuild is not None: refRootfileBuildFQN = os.path.join( referenceBuildWorkflowFQNPath, refRootfileBuild) print "comparing file ", rootfileBuildFQN, " with ", refRootfileBuildFQN outputReportName = workflow outputReportDir = os.path.join(buildNameFQNPath, outputReportBase, outputReportName) increment = 0 while os.path.exists(outputReportDir): increment = increment + 1 outputReportDir = outputReportDir + '-' + str( increment) os.makedirs(outputReportDir) waitThreads(wThreads, config.MachineCPUCount) thrd = Thread( target=makeComparison, args=(buildNameFQNPath, buildArchitecture, no_successes, rootfileBuildFQN, refRootfileBuildFQN, outputReportDir, threshold, statTest, doPngs, doComparison, doReport, workflowDir, workflowFQNParentPath, black_list, success_percentage, stamp, rel, outputReportName, buildName, workflow, referenceBuildName, report_path, report_relative_path, dryRun)) wThreads[thrd] = 1 thrd.start() time.sleep(2) else: print '%s folder doesn\'t contain the referencing DQM harvested root file.' % workflowDir else: print '%s folder doesn\'t contain the DQM harvested root file.' % workflowDir waitThreads(wThreads) #if os.path.exists(outputReportDir): # print 'Deleting temporary build report dir output' # doCmd.doCmd('rm -rf %s' % outputReportDir, dryRun) print "Done" return