def run(self):
IBThreadBase.run(self)
try:
dqmrefargs = os.environ['CMS_PATH'] + "/" + os.environ[
'SCRAM_ARCH'] + "/cms/cmssw-dqm-reference-deployer/" + config.getDQMReferenceBuild(
) + "/etc/runTheMatrix.args"
if not os.path.exists(dqmrefargs):
print 'ERROR: DQM Ref data is not available:' + dqmrefargs
return
inArgs = open(dqmrefargs, 'r')
thrds = []
runCmd("rm -rf " + self.startDir)
for line in inArgs.readlines():
thrd = RunMatirxCmd(self.startDir + "/wfs_" + str(len(thrds)),
line.rstrip() + " -j 2")
thrd.start()
thrds.append(thrd)
inArgs.close()
for thrd in thrds:
thrd.join()
for rfile in glob.glob(self.startDir +
"/wfs_*/*HARVEST*/DQM_V*.root"):
wfdir = self.startDir + "/data/" + re.sub(
"/DQM_V.+\.root", "",
re.sub(self.startDir + "/wfs_\d+/", "", rfile))
cmd = "mkdir -p " + wfdir + " ; cp " + rfile + " " + wfdir
runCmd(cmd)
runCmd("rm -f " + self.startDir + "/wfs_*/*HARVEST*/*.root")
except Exception, e:
print "ERROR during GenerateDQMRef : caught exception: " + str(e)
pass
def run(self):
IBThreadBase.run(self)
try:
dqmrefargs=os.environ['CMS_PATH']+"/"+os.environ['SCRAM_ARCH']+"/cms/cmssw-dqm-reference-deployer/"+config.getDQMReferenceBuild()+"/etc/runTheMatrix.args"
if not os.path.exists(dqmrefargs):
print 'ERROR: DQM Ref data is not available:'+dqmrefargs
return
inArgs = open (dqmrefargs, 'r')
thrds = []
runCmd("rm -rf "+self.startDir)
for line in inArgs.readlines():
thrd = RunMatirxCmd(self.startDir+"/wfs_"+str(len(thrds)),line.rstrip()+" -j 2")
thrd.start()
thrds.append(thrd)
inArgs.close()
for thrd in thrds: thrd.join()
for rfile in glob.glob(self.startDir+"/wfs_*/*HARVEST*/DQM_V*.root"):
wfdir = self.startDir+"/data/"+re.sub("/DQM_V.+\.root","",re.sub(self.startDir+"/wfs_\d+/","",rfile))
cmd = "mkdir -p "+wfdir+" ; cp "+rfile+" "+wfdir
runCmd(cmd)
runCmd("rm -f "+self.startDir+"/wfs_*/*HARVEST*/*.root")
except Exception, e :
print "ERROR during GenerateDQMRef : caught exception: " + str(e)
pass
def main():
parser = OptionParser()
usage = "usage: %prog [option1] arg1 [option2] arg2 ... [optionN] argN"
parser = OptionParser(usage=usage)
parser.add_option("-b", "--buildDir",
dest="buildDir",
help="build directory")
parser.add_option("-r", "--releaseDir",
dest="releaseDir",
help="release directory")
parser.add_option("-d", "--dryRun",
action="store_true", dest="dryRun",
help="is a dry run?")
parser.add_option("-p", "--platform",
dest="platform",
help="used architecture")
parser.add_option("-c", "--relCycle",
dest="relCycle",
help="Release cycle")
parser.add_option("-s", "--stamp",
dest="stamp",
help="Release stamp")
parser.add_option("-t", "--relTag",
dest="relTag",
help="Release tag")
parser.add_option("-u", "--custom_ib",
action="store_true",
dest="custom_ib",
help="Sets the specific configuration for running DQM comparison for Custom IBs.")
parser.add_option("-e", "--report_path",
dest="report_path",
help="Path to report. Optional value")
parser.add_option("-o", "--report_relative_path",
dest="report_relative_path",
help="Relative Path to report. The value is used for compiling report web page url. Optional value")
(options, args) = parser.parse_args()
dryRun = False
if not options.dryRun == None:
dryRun = options.dryRun
custom_ib = False
if not options.custom_ib == None:
custom_ib = options.custom_ib
if options.buildDir == None:
print "The Integration Build directory is not specified"
return errno.EINVAL
buildDir = options.buildDir
report_path = options.report_path
report_relative_path = options.report_relative_path
while buildDir.endswith('/'):
buildDir = buildDir[:(len(buildDir)-1)]
(head, stamp) = os.path.split(buildDir)
head.strip() #hack, don't show never read warning
if options.stamp == None:
stamp = str(stamp).strip()
else:
stamp = options.stamp
if options.relTag == None:
buildName = getBuildName(buildDir)
else:
buildName = options.relTag
if options.platform == None:
buildArchitecture = getBuildArchitecture(buildName, buildDir)
else:
buildArchitecture = options.platform
if options.relCycle == None:
rel = getRelVersion(buildName)
else:
rel = options.relCycle
intBldFQNPath = os.path.dirname(os.path.dirname(buildDir))
if not options.releaseDir == None:
buildNameFQNPath = options.releaseDir
else:
buildNameFQNPath = os.path.join(buildDir, buildName)
if not os.path.exists(buildNameFQNPath):
print 'Build is not found in common path %s' % str(buildNameFQNPath)
buildNameFQNPath = os.path.join(intBldFQNPath, 'cms', buildArchitecture, 'cms', 'cmssw', buildName)
print 'Looking for the build in %s' % str(buildNameFQNPath)
if not os.path.exists(buildNameFQNPath):
print 'The build location cannot be found, terminating ...'
return errno.ENOENT
import config
config.setDefaults(rel)
relMonParameters = config.Configuration[rel]['RelMonParameters']
if custom_ib:
deployer_spec_file = os.path.join( buildNameFQNPath, 'tmp', 'tmpspec-cmssw-dqm-reference-deployer')
referenceBuildName = config.getDQMReferenceBuild(deployer_spec_file)
else:
referenceBuildName = config.getDQMReferenceBuild()
if not referenceBuildName:
print "Reference build doesn't exist in configuration, exiting comparison ..."
return errno.ENOENT
referenceBuildFQNPath = getReferenceBuildFQNPath(config, rel, referenceBuildName, buildArchitecture);
if not os.path.exists(referenceBuildFQNPath):
referenceBuildFQNPath2 = os.path.join(buildNameFQNPath, buildArchitecture, 'cms', 'cmssw-dqm-reference-deployer', referenceBuildName, 'data')
print "Reference build path %s doesn't exist, will search in %s" % (referenceBuildFQNPath,referenceBuildFQNPath2,)
referenceBuildFQNPath = referenceBuildFQNPath2
(referenceBuildName, referenceBuildArchitecture) = searchReferenceBuildNameAndArchitectureInPath(referenceBuildFQNPath)
threshold = getRelMonValue(relMonParameters,'threshold')
statTest = getRelMonValue(relMonParameters,'statTest')
doPngs = getRelMonValue(relMonParameters,'doPngs')
doComparison = getRelMonValue(relMonParameters,'doComparison')
doReport = getRelMonValue(relMonParameters,'doReport')
no_successes = getRelMonValue(relMonParameters,'no_successes')
black_list = getRelMonValue(relMonParameters,'black_list')
success_percentage = getRelMonValue(relMonParameters, 'success_percentage')
workflowFQNParentPath = os.path.join(buildNameFQNPath, workflowDataRelativePath)
if not os.path.exists(workflowFQNParentPath):
workflowFQNParentPath2 = os.path.join(buildNameFQNPath, workflowDataRelativePath_tests)
print 'The workflow location %s cannot be found, will search in %s' % (workflowFQNParentPath, workflowFQNParentPath2,)
workflowFQNParentPath = workflowFQNParentPath2
if not os.path.exists(workflowFQNParentPath):
print 'The workflow location %s cannot be found, terminating ...' % workflowFQNParentPath
return errno.ENOENT
dirs = os.listdir(workflowFQNParentPath)
wThreads = {}
for workflowDir in dirs:
if 'HARVEST' in workflowDir:
workflow = getWorkflowNumber(workflowDir)
workflowDirFQN = os.path.join(workflowFQNParentPath, workflowDir)
dqmHarvestedRootFile = getDQMHarvestedRootFile(workflowDirFQN)
if dqmHarvestedRootFile is not None:
rootfileBuildFQN = os.path.join(workflowDirFQN, dqmHarvestedRootFile)
## reference build root file
referenceWorkflows = os.listdir(referenceBuildFQNPath)
referenceBuildWorkflowFQNPath = None
for referenceWorkflow in referenceWorkflows:
if referenceWorkflow.startswith(str(workflow)):
referenceBuildWorkflowFQNPath = os.path.join(referenceBuildFQNPath, referenceWorkflow)
break
if referenceBuildWorkflowFQNPath == None:
print "Workflow ", str(workflow), " doesn't exist in the reference build folder ", str(referenceBuildFQNPath)
continue #go to next workflow iw current one is missing
refRootfileBuild = getDQMHarvestedRootFile(referenceBuildWorkflowFQNPath)
if refRootfileBuild is not None:
refRootfileBuildFQN = os.path.join(referenceBuildWorkflowFQNPath, refRootfileBuild)
print "comparing file ", rootfileBuildFQN, " with ", refRootfileBuildFQN
outputReportName = workflow
outputReportDir = os.path.join(buildNameFQNPath, outputReportBase, outputReportName)
increment = 0
while os.path.exists(outputReportDir):
increment = increment + 1
outputReportDir = outputReportDir + '-' + str(increment)
os.makedirs(outputReportDir)
waitThreads(wThreads,config.MachineCPUCount)
thrd = Thread(target=makeComparison, args=(buildNameFQNPath,
buildArchitecture,
no_successes,
rootfileBuildFQN,
refRootfileBuildFQN,
outputReportDir,
threshold,
statTest,
doPngs,
doComparison,
doReport,
workflowDir,
workflowFQNParentPath,
black_list,
success_percentage,
stamp,
rel,
outputReportName,
buildName,
workflow,
referenceBuildName,
report_path,
report_relative_path,
dryRun))
wThreads[thrd]=1
thrd.start()
time.sleep(2)
else:
print '%s folder doesn\'t contain the referencing DQM harvested root file.' % workflowDir
else:
print '%s folder doesn\'t contain the DQM harvested root file.' % workflowDir
waitThreads(wThreads)
#if os.path.exists(outputReportDir):
# print 'Deleting temporary build report dir output'
# doCmd.doCmd('rm -rf %s' % outputReportDir, dryRun)
print "Done"
return
def main():
parser = OptionParser()
usage = "usage: %prog [option1] arg1 [option2] arg2 ... [optionN] argN"
parser = OptionParser(usage=usage)
parser.add_option("-b",
"--buildDir",
dest="buildDir",
help="build directory")
parser.add_option("-r",
"--releaseDir",
dest="releaseDir",
help="release directory")
parser.add_option("-d",
"--dryRun",
action="store_true",
dest="dryRun",
help="is a dry run?")
parser.add_option("-p",
"--platform",
dest="platform",
help="used architecture")
parser.add_option("-c",
"--relCycle",
dest="relCycle",
help="Release cycle")
parser.add_option("-s", "--stamp", dest="stamp", help="Release stamp")
parser.add_option("-t", "--relTag", dest="relTag", help="Release tag")
parser.add_option(
"-u",
"--custom_ib",
action="store_true",
dest="custom_ib",
help=
"Sets the specific configuration for running DQM comparison for Custom IBs."
)
parser.add_option("-e",
"--report_path",
dest="report_path",
help="Path to report. Optional value")
parser.add_option(
"-o",
"--report_relative_path",
dest="report_relative_path",
help=
"Relative Path to report. The value is used for compiling report web page url. Optional value"
)
(options, args) = parser.parse_args()
dryRun = False
if not options.dryRun == None:
dryRun = options.dryRun
custom_ib = False
if not options.custom_ib == None:
custom_ib = options.custom_ib
if options.buildDir == None:
print "The Integration Build directory is not specified"
return errno.EINVAL
buildDir = options.buildDir
report_path = options.report_path
report_relative_path = options.report_relative_path
while buildDir.endswith('/'):
buildDir = buildDir[:(len(buildDir) - 1)]
(head, stamp) = os.path.split(buildDir)
head.strip() #hack, don't show never read warning
if options.stamp == None:
stamp = str(stamp).strip()
else:
stamp = options.stamp
if options.relTag == None:
buildName = getBuildName(buildDir)
else:
buildName = options.relTag
if options.platform == None:
buildArchitecture = getBuildArchitecture(buildName, buildDir)
else:
buildArchitecture = options.platform
if options.relCycle == None:
rel = getRelVersion(buildName)
else:
rel = options.relCycle
intBldFQNPath = os.path.dirname(os.path.dirname(buildDir))
if not options.releaseDir == None:
buildNameFQNPath = options.releaseDir
else:
buildNameFQNPath = os.path.join(buildDir, buildName)
if not os.path.exists(buildNameFQNPath):
print 'Build is not found in common path %s' % str(buildNameFQNPath)
buildNameFQNPath = os.path.join(intBldFQNPath, 'cms',
buildArchitecture, 'cms', 'cmssw',
buildName)
print 'Looking for the build in %s' % str(buildNameFQNPath)
if not os.path.exists(buildNameFQNPath):
print 'The build location cannot be found, terminating ...'
return errno.ENOENT
import config
config.setDefaults(rel)
relMonParameters = config.Configuration[rel]['RelMonParameters']
if custom_ib:
deployer_spec_file = os.path.join(
buildNameFQNPath, 'tmp', 'tmpspec-cmssw-dqm-reference-deployer')
referenceBuildName = config.getDQMReferenceBuild(deployer_spec_file)
else:
referenceBuildName = config.getDQMReferenceBuild()
if not referenceBuildName:
print "Reference build doesn't exist in configuration, exiting comparison ..."
return errno.ENOENT
referenceBuildFQNPath = getReferenceBuildFQNPath(config, rel,
referenceBuildName,
buildArchitecture)
if not os.path.exists(referenceBuildFQNPath):
referenceBuildFQNPath2 = os.path.join(buildNameFQNPath,
buildArchitecture, 'cms',
'cmssw-dqm-reference-deployer',
referenceBuildName, 'data')
print "Reference build path %s doesn't exist, will search in %s" % (
referenceBuildFQNPath,
referenceBuildFQNPath2,
)
referenceBuildFQNPath = referenceBuildFQNPath2
(referenceBuildName, referenceBuildArchitecture
) = searchReferenceBuildNameAndArchitectureInPath(referenceBuildFQNPath)
threshold = getRelMonValue(relMonParameters, 'threshold')
statTest = getRelMonValue(relMonParameters, 'statTest')
doPngs = getRelMonValue(relMonParameters, 'doPngs')
doComparison = getRelMonValue(relMonParameters, 'doComparison')
doReport = getRelMonValue(relMonParameters, 'doReport')
no_successes = getRelMonValue(relMonParameters, 'no_successes')
black_list = getRelMonValue(relMonParameters, 'black_list')
success_percentage = getRelMonValue(relMonParameters, 'success_percentage')
workflowFQNParentPath = os.path.join(buildNameFQNPath,
workflowDataRelativePath)
if not os.path.exists(workflowFQNParentPath):
workflowFQNParentPath2 = os.path.join(buildNameFQNPath,
workflowDataRelativePath_tests)
print 'The workflow location %s cannot be found, will search in %s' % (
workflowFQNParentPath,
workflowFQNParentPath2,
)
workflowFQNParentPath = workflowFQNParentPath2
if not os.path.exists(workflowFQNParentPath):
print 'The workflow location %s cannot be found, terminating ...' % workflowFQNParentPath
return errno.ENOENT
dirs = os.listdir(workflowFQNParentPath)
wThreads = {}
for workflowDir in dirs:
if 'HARVEST' in workflowDir:
workflow = getWorkflowNumber(workflowDir)
workflowDirFQN = os.path.join(workflowFQNParentPath, workflowDir)
dqmHarvestedRootFile = getDQMHarvestedRootFile(workflowDirFQN)
if dqmHarvestedRootFile is not None:
rootfileBuildFQN = os.path.join(workflowDirFQN,
dqmHarvestedRootFile)
## reference build root file
referenceWorkflows = os.listdir(referenceBuildFQNPath)
referenceBuildWorkflowFQNPath = None
for referenceWorkflow in referenceWorkflows:
if referenceWorkflow.startswith(str(workflow)):
referenceBuildWorkflowFQNPath = os.path.join(
referenceBuildFQNPath, referenceWorkflow)
break
if referenceBuildWorkflowFQNPath == None:
print "Workflow ", str(
workflow
), " doesn't exist in the reference build folder ", str(
referenceBuildFQNPath)
continue #go to next workflow iw current one is missing
refRootfileBuild = getDQMHarvestedRootFile(
referenceBuildWorkflowFQNPath)
if refRootfileBuild is not None:
refRootfileBuildFQN = os.path.join(
referenceBuildWorkflowFQNPath, refRootfileBuild)
print "comparing file ", rootfileBuildFQN, " with ", refRootfileBuildFQN
outputReportName = workflow
outputReportDir = os.path.join(buildNameFQNPath,
outputReportBase,
outputReportName)
increment = 0
while os.path.exists(outputReportDir):
increment = increment + 1
outputReportDir = outputReportDir + '-' + str(
increment)
os.makedirs(outputReportDir)
waitThreads(wThreads, config.MachineCPUCount)
thrd = Thread(
target=makeComparison,
args=(buildNameFQNPath, buildArchitecture,
no_successes, rootfileBuildFQN,
refRootfileBuildFQN, outputReportDir, threshold,
statTest, doPngs, doComparison, doReport,
workflowDir, workflowFQNParentPath, black_list,
success_percentage, stamp, rel, outputReportName,
buildName, workflow, referenceBuildName,
report_path, report_relative_path, dryRun))
wThreads[thrd] = 1
thrd.start()
time.sleep(2)
else:
print '%s folder doesn\'t contain the referencing DQM harvested root file.' % workflowDir
else:
print '%s folder doesn\'t contain the DQM harvested root file.' % workflowDir
waitThreads(wThreads)
#if os.path.exists(outputReportDir):
# print 'Deleting temporary build report dir output'
# doCmd.doCmd('rm -rf %s' % outputReportDir, dryRun)
print "Done"
return
