def run(self): IBThreadBase.run(self) try: dqmrefargs = os.environ['CMS_PATH'] + "/" + os.environ[ 'SCRAM_ARCH'] + "/cms/cmssw-dqm-reference-deployer/" + config.getDQMReferenceBuild( ) + "/etc/runTheMatrix.args" if not os.path.exists(dqmrefargs): print 'ERROR: DQM Ref data is not available:' + dqmrefargs return inArgs = open(dqmrefargs, 'r') thrds = [] runCmd("rm -rf " + self.startDir) for line in inArgs.readlines(): thrd = RunMatirxCmd(self.startDir + "/wfs_" + str(len(thrds)), line.rstrip() + " -j 2") thrd.start() thrds.append(thrd) inArgs.close() for thrd in thrds: thrd.join() for rfile in glob.glob(self.startDir + "/wfs_*/*HARVEST*/DQM_V*.root"): wfdir = self.startDir + "/data/" + re.sub( "/DQM_V.+\.root", "", re.sub(self.startDir + "/wfs_\d+/", "", rfile)) cmd = "mkdir -p " + wfdir + " ; cp " + rfile + " " + wfdir runCmd(cmd) runCmd("rm -f " + self.startDir + "/wfs_*/*HARVEST*/*.root") except Exception, e: print "ERROR during GenerateDQMRef : caught exception: " + str(e) pass
def run(self): IBThreadBase.run(self) try: dqmrefargs=os.environ['CMS_PATH']+"/"+os.environ['SCRAM_ARCH']+"/cms/cmssw-dqm-reference-deployer/"+config.getDQMReferenceBuild()+"/etc/runTheMatrix.args" if not os.path.exists(dqmrefargs): print 'ERROR: DQM Ref data is not available:'+dqmrefargs return inArgs = open (dqmrefargs, 'r') thrds = [] runCmd("rm -rf "+self.startDir) for line in inArgs.readlines(): thrd = RunMatirxCmd(self.startDir+"/wfs_"+str(len(thrds)),line.rstrip()+" -j 2") thrd.start() thrds.append(thrd) inArgs.close() for thrd in thrds: thrd.join() for rfile in glob.glob(self.startDir+"/wfs_*/*HARVEST*/DQM_V*.root"): wfdir = self.startDir+"/data/"+re.sub("/DQM_V.+\.root","",re.sub(self.startDir+"/wfs_\d+/","",rfile)) cmd = "mkdir -p "+wfdir+" ; cp "+rfile+" "+wfdir runCmd(cmd) runCmd("rm -f "+self.startDir+"/wfs_*/*HARVEST*/*.root") except Exception, e : print "ERROR during GenerateDQMRef : caught exception: " + str(e) pass
def main(): parser = OptionParser() usage = "usage: %prog [option1] arg1 [option2] arg2 ... [optionN] argN" parser = OptionParser(usage=usage) parser.add_option("-b", "--buildDir", dest="buildDir", help="build directory") parser.add_option("-r", "--releaseDir", dest="releaseDir", help="release directory") parser.add_option("-d", "--dryRun", action="store_true", dest="dryRun", help="is a dry run?") parser.add_option("-p", "--platform", dest="platform", help="used architecture") parser.add_option("-c", "--relCycle", dest="relCycle", help="Release cycle") parser.add_option("-s", "--stamp", dest="stamp", help="Release stamp") parser.add_option("-t", "--relTag", dest="relTag", help="Release tag") parser.add_option("-u", "--custom_ib", action="store_true", dest="custom_ib", help="Sets the specific configuration for running DQM comparison for Custom IBs.") parser.add_option("-e", "--report_path", dest="report_path", help="Path to report. Optional value") parser.add_option("-o", "--report_relative_path", dest="report_relative_path", help="Relative Path to report. The value is used for compiling report web page url. Optional value") (options, args) = parser.parse_args() dryRun = False if not options.dryRun == None: dryRun = options.dryRun custom_ib = False if not options.custom_ib == None: custom_ib = options.custom_ib if options.buildDir == None: print "The Integration Build directory is not specified" return errno.EINVAL buildDir = options.buildDir report_path = options.report_path report_relative_path = options.report_relative_path while buildDir.endswith('/'): buildDir = buildDir[:(len(buildDir)-1)] (head, stamp) = os.path.split(buildDir) head.strip() #hack, don't show never read warning if options.stamp == None: stamp = str(stamp).strip() else: stamp = options.stamp if options.relTag == None: buildName = getBuildName(buildDir) else: buildName = options.relTag if options.platform == None: buildArchitecture = getBuildArchitecture(buildName, buildDir) else: buildArchitecture = options.platform if options.relCycle == None: rel = getRelVersion(buildName) else: rel = options.relCycle intBldFQNPath = os.path.dirname(os.path.dirname(buildDir)) if not options.releaseDir == None: buildNameFQNPath = options.releaseDir else: buildNameFQNPath = os.path.join(buildDir, buildName) if not os.path.exists(buildNameFQNPath): print 'Build is not found in common path %s' % str(buildNameFQNPath) buildNameFQNPath = os.path.join(intBldFQNPath, 'cms', buildArchitecture, 'cms', 'cmssw', buildName) print 'Looking for the build in %s' % str(buildNameFQNPath) if not os.path.exists(buildNameFQNPath): print 'The build location cannot be found, terminating ...' return errno.ENOENT import config config.setDefaults(rel) relMonParameters = config.Configuration[rel]['RelMonParameters'] if custom_ib: deployer_spec_file = os.path.join( buildNameFQNPath, 'tmp', 'tmpspec-cmssw-dqm-reference-deployer') referenceBuildName = config.getDQMReferenceBuild(deployer_spec_file) else: referenceBuildName = config.getDQMReferenceBuild() if not referenceBuildName: print "Reference build doesn't exist in configuration, exiting comparison ..." return errno.ENOENT referenceBuildFQNPath = getReferenceBuildFQNPath(config, rel, referenceBuildName, buildArchitecture); if not os.path.exists(referenceBuildFQNPath): referenceBuildFQNPath2 = os.path.join(buildNameFQNPath, buildArchitecture, 'cms', 'cmssw-dqm-reference-deployer', referenceBuildName, 'data') print "Reference build path %s doesn't exist, will search in %s" % (referenceBuildFQNPath,referenceBuildFQNPath2,) referenceBuildFQNPath = referenceBuildFQNPath2 (referenceBuildName, referenceBuildArchitecture) = searchReferenceBuildNameAndArchitectureInPath(referenceBuildFQNPath) threshold = getRelMonValue(relMonParameters,'threshold') statTest = getRelMonValue(relMonParameters,'statTest') doPngs = getRelMonValue(relMonParameters,'doPngs') doComparison = getRelMonValue(relMonParameters,'doComparison') doReport = getRelMonValue(relMonParameters,'doReport') no_successes = getRelMonValue(relMonParameters,'no_successes') black_list = getRelMonValue(relMonParameters,'black_list') success_percentage = getRelMonValue(relMonParameters, 'success_percentage') workflowFQNParentPath = os.path.join(buildNameFQNPath, workflowDataRelativePath) if not os.path.exists(workflowFQNParentPath): workflowFQNParentPath2 = os.path.join(buildNameFQNPath, workflowDataRelativePath_tests) print 'The workflow location %s cannot be found, will search in %s' % (workflowFQNParentPath, workflowFQNParentPath2,) workflowFQNParentPath = workflowFQNParentPath2 if not os.path.exists(workflowFQNParentPath): print 'The workflow location %s cannot be found, terminating ...' % workflowFQNParentPath return errno.ENOENT dirs = os.listdir(workflowFQNParentPath) wThreads = {} for workflowDir in dirs: if 'HARVEST' in workflowDir: workflow = getWorkflowNumber(workflowDir) workflowDirFQN = os.path.join(workflowFQNParentPath, workflowDir) dqmHarvestedRootFile = getDQMHarvestedRootFile(workflowDirFQN) if dqmHarvestedRootFile is not None: rootfileBuildFQN = os.path.join(workflowDirFQN, dqmHarvestedRootFile) ## reference build root file referenceWorkflows = os.listdir(referenceBuildFQNPath) referenceBuildWorkflowFQNPath = None for referenceWorkflow in referenceWorkflows: if referenceWorkflow.startswith(str(workflow)): referenceBuildWorkflowFQNPath = os.path.join(referenceBuildFQNPath, referenceWorkflow) break if referenceBuildWorkflowFQNPath == None: print "Workflow ", str(workflow), " doesn't exist in the reference build folder ", str(referenceBuildFQNPath) continue #go to next workflow iw current one is missing refRootfileBuild = getDQMHarvestedRootFile(referenceBuildWorkflowFQNPath) if refRootfileBuild is not None: refRootfileBuildFQN = os.path.join(referenceBuildWorkflowFQNPath, refRootfileBuild) print "comparing file ", rootfileBuildFQN, " with ", refRootfileBuildFQN outputReportName = workflow outputReportDir = os.path.join(buildNameFQNPath, outputReportBase, outputReportName) increment = 0 while os.path.exists(outputReportDir): increment = increment + 1 outputReportDir = outputReportDir + '-' + str(increment) os.makedirs(outputReportDir) waitThreads(wThreads,config.MachineCPUCount) thrd = Thread(target=makeComparison, args=(buildNameFQNPath, buildArchitecture, no_successes, rootfileBuildFQN, refRootfileBuildFQN, outputReportDir, threshold, statTest, doPngs, doComparison, doReport, workflowDir, workflowFQNParentPath, black_list, success_percentage, stamp, rel, outputReportName, buildName, workflow, referenceBuildName, report_path, report_relative_path, dryRun)) wThreads[thrd]=1 thrd.start() time.sleep(2) else: print '%s folder doesn\'t contain the referencing DQM harvested root file.' % workflowDir else: print '%s folder doesn\'t contain the DQM harvested root file.' % workflowDir waitThreads(wThreads) #if os.path.exists(outputReportDir): # print 'Deleting temporary build report dir output' # doCmd.doCmd('rm -rf %s' % outputReportDir, dryRun) print "Done" return
def main(): parser = OptionParser() usage = "usage: %prog [option1] arg1 [option2] arg2 ... [optionN] argN" parser = OptionParser(usage=usage) parser.add_option("-b", "--buildDir", dest="buildDir", help="build directory") parser.add_option("-r", "--releaseDir", dest="releaseDir", help="release directory") parser.add_option("-d", "--dryRun", action="store_true", dest="dryRun", help="is a dry run?") parser.add_option("-p", "--platform", dest="platform", help="used architecture") parser.add_option("-c", "--relCycle", dest="relCycle", help="Release cycle") parser.add_option("-s", "--stamp", dest="stamp", help="Release stamp") parser.add_option("-t", "--relTag", dest="relTag", help="Release tag") parser.add_option( "-u", "--custom_ib", action="store_true", dest="custom_ib", help= "Sets the specific configuration for running DQM comparison for Custom IBs." ) parser.add_option("-e", "--report_path", dest="report_path", help="Path to report. Optional value") parser.add_option( "-o", "--report_relative_path", dest="report_relative_path", help= "Relative Path to report. The value is used for compiling report web page url. Optional value" ) (options, args) = parser.parse_args() dryRun = False if not options.dryRun == None: dryRun = options.dryRun custom_ib = False if not options.custom_ib == None: custom_ib = options.custom_ib if options.buildDir == None: print "The Integration Build directory is not specified" return errno.EINVAL buildDir = options.buildDir report_path = options.report_path report_relative_path = options.report_relative_path while buildDir.endswith('/'): buildDir = buildDir[:(len(buildDir) - 1)] (head, stamp) = os.path.split(buildDir) head.strip() #hack, don't show never read warning if options.stamp == None: stamp = str(stamp).strip() else: stamp = options.stamp if options.relTag == None: buildName = getBuildName(buildDir) else: buildName = options.relTag if options.platform == None: buildArchitecture = getBuildArchitecture(buildName, buildDir) else: buildArchitecture = options.platform if options.relCycle == None: rel = getRelVersion(buildName) else: rel = options.relCycle intBldFQNPath = os.path.dirname(os.path.dirname(buildDir)) if not options.releaseDir == None: buildNameFQNPath = options.releaseDir else: buildNameFQNPath = os.path.join(buildDir, buildName) if not os.path.exists(buildNameFQNPath): print 'Build is not found in common path %s' % str(buildNameFQNPath) buildNameFQNPath = os.path.join(intBldFQNPath, 'cms', buildArchitecture, 'cms', 'cmssw', buildName) print 'Looking for the build in %s' % str(buildNameFQNPath) if not os.path.exists(buildNameFQNPath): print 'The build location cannot be found, terminating ...' return errno.ENOENT import config config.setDefaults(rel) relMonParameters = config.Configuration[rel]['RelMonParameters'] if custom_ib: deployer_spec_file = os.path.join( buildNameFQNPath, 'tmp', 'tmpspec-cmssw-dqm-reference-deployer') referenceBuildName = config.getDQMReferenceBuild(deployer_spec_file) else: referenceBuildName = config.getDQMReferenceBuild() if not referenceBuildName: print "Reference build doesn't exist in configuration, exiting comparison ..." return errno.ENOENT referenceBuildFQNPath = getReferenceBuildFQNPath(config, rel, referenceBuildName, buildArchitecture) if not os.path.exists(referenceBuildFQNPath): referenceBuildFQNPath2 = os.path.join(buildNameFQNPath, buildArchitecture, 'cms', 'cmssw-dqm-reference-deployer', referenceBuildName, 'data') print "Reference build path %s doesn't exist, will search in %s" % ( referenceBuildFQNPath, referenceBuildFQNPath2, ) referenceBuildFQNPath = referenceBuildFQNPath2 (referenceBuildName, referenceBuildArchitecture ) = searchReferenceBuildNameAndArchitectureInPath(referenceBuildFQNPath) threshold = getRelMonValue(relMonParameters, 'threshold') statTest = getRelMonValue(relMonParameters, 'statTest') doPngs = getRelMonValue(relMonParameters, 'doPngs') doComparison = getRelMonValue(relMonParameters, 'doComparison') doReport = getRelMonValue(relMonParameters, 'doReport') no_successes = getRelMonValue(relMonParameters, 'no_successes') black_list = getRelMonValue(relMonParameters, 'black_list') success_percentage = getRelMonValue(relMonParameters, 'success_percentage') workflowFQNParentPath = os.path.join(buildNameFQNPath, workflowDataRelativePath) if not os.path.exists(workflowFQNParentPath): workflowFQNParentPath2 = os.path.join(buildNameFQNPath, workflowDataRelativePath_tests) print 'The workflow location %s cannot be found, will search in %s' % ( workflowFQNParentPath, workflowFQNParentPath2, ) workflowFQNParentPath = workflowFQNParentPath2 if not os.path.exists(workflowFQNParentPath): print 'The workflow location %s cannot be found, terminating ...' % workflowFQNParentPath return errno.ENOENT dirs = os.listdir(workflowFQNParentPath) wThreads = {} for workflowDir in dirs: if 'HARVEST' in workflowDir: workflow = getWorkflowNumber(workflowDir) workflowDirFQN = os.path.join(workflowFQNParentPath, workflowDir) dqmHarvestedRootFile = getDQMHarvestedRootFile(workflowDirFQN) if dqmHarvestedRootFile is not None: rootfileBuildFQN = os.path.join(workflowDirFQN, dqmHarvestedRootFile) ## reference build root file referenceWorkflows = os.listdir(referenceBuildFQNPath) referenceBuildWorkflowFQNPath = None for referenceWorkflow in referenceWorkflows: if referenceWorkflow.startswith(str(workflow)): referenceBuildWorkflowFQNPath = os.path.join( referenceBuildFQNPath, referenceWorkflow) break if referenceBuildWorkflowFQNPath == None: print "Workflow ", str( workflow ), " doesn't exist in the reference build folder ", str( referenceBuildFQNPath) continue #go to next workflow iw current one is missing refRootfileBuild = getDQMHarvestedRootFile( referenceBuildWorkflowFQNPath) if refRootfileBuild is not None: refRootfileBuildFQN = os.path.join( referenceBuildWorkflowFQNPath, refRootfileBuild) print "comparing file ", rootfileBuildFQN, " with ", refRootfileBuildFQN outputReportName = workflow outputReportDir = os.path.join(buildNameFQNPath, outputReportBase, outputReportName) increment = 0 while os.path.exists(outputReportDir): increment = increment + 1 outputReportDir = outputReportDir + '-' + str( increment) os.makedirs(outputReportDir) waitThreads(wThreads, config.MachineCPUCount) thrd = Thread( target=makeComparison, args=(buildNameFQNPath, buildArchitecture, no_successes, rootfileBuildFQN, refRootfileBuildFQN, outputReportDir, threshold, statTest, doPngs, doComparison, doReport, workflowDir, workflowFQNParentPath, black_list, success_percentage, stamp, rel, outputReportName, buildName, workflow, referenceBuildName, report_path, report_relative_path, dryRun)) wThreads[thrd] = 1 thrd.start() time.sleep(2) else: print '%s folder doesn\'t contain the referencing DQM harvested root file.' % workflowDir else: print '%s folder doesn\'t contain the DQM harvested root file.' % workflowDir waitThreads(wThreads) #if os.path.exists(outputReportDir): # print 'Deleting temporary build report dir output' # doCmd.doCmd('rm -rf %s' % outputReportDir, dryRun) print "Done" return