def __init__(self, buildDir, releaseDir, appset, doInstall=True):
BuilderBase.__init__(self)
os.environ['STAGE_SVCCLASS']='t1transfer'
os.environ['STAGE_HOST']='castorcms'
os.environ['STAGER_TRACE']='3'
self.doInstall=doInstall
self.buildDir = buildDir
self.appset = appset
self.updateTimeStamp(self.buildDir)
self.cmsswBuildDir = releaseDir
self.release = os.path.basename(releaseDir)
self.relTag = self.release
self.relCycle = self.release.replace("CMSSW_","").split("_X_",1)[0].replace("_",".")
day,hour = self.stamp.split('-')
self.threadList = {}
self.maxThreads = 2 # one for unit-tests, one for addOnTests
self.doRelVal = True
self.logger = None
self.RelValArgs = None
config.setDefaults(self.relCycle)
try:
self.doRelVal = config.Configuration[self.relCycle]['runRelVal']
self.RelValArgs = config.Configuration[self.relCycle]['RelValArgs']
except:
pass
return
def __init__(self, buildDir, releaseDir, appset, doInstall=True):
BuilderBase.__init__(self)
os.environ['STAGE_SVCCLASS'] = 't1transfer'
os.environ['STAGE_HOST'] = 'castorcms'
os.environ['STAGER_TRACE'] = '3'
self.doInstall = doInstall
self.buildDir = buildDir
self.appset = appset
self.updateTimeStamp(self.buildDir)
self.cmsswBuildDir = releaseDir
self.release = os.path.basename(releaseDir)
self.relTag = self.release
self.relCycle = self.release.replace("CMSSW_",
"").split("_X_",
1)[0].replace("_", ".")
day, hour = self.stamp.split('-')
self.threadList = {}
self.maxThreads = 2 # one for unit-tests, one for addOnTests
self.doRelVal = True
self.logger = None
self.RelValArgs = None
config.setDefaults(self.relCycle)
try:
self.doRelVal = config.Configuration[self.relCycle]['runRelVal']
self.RelValArgs = config.Configuration[self.relCycle]['RelValArgs']
except:
pass
return
def __init__(self, dayIn=None, relCycIn=None, dryRunIn=False,doInstall=True) :
BuilderBase.__init__(self)
self.day = dayIn
if not self.day :
self.day = time.strftime("%a").lower()
self.dryRun = dryRunIn
self.topDir = os.path.join(self.installDir, self.plat)
self.relCycle = relCycIn
self.doInstall = doInstall
self.cand = None
config.setDefaults (relCycIn)
return
def main():
import getopt
options = sys.argv[1:]
try:
opts, args = getopt.getopt(
options, 'h',
['help', 'buildDir=', 'release=', 'tag=', 'dryRun', 'cycle='])
except getopt.GetoptError:
usage()
sys.exit(-2)
buildDir = None
rel = None
tag = None
dryRun = False
cycle = None
for o, a in opts:
if o in ('-h', '--help'):
usage()
sys.exit()
if o in ('--buildDir', ):
buildDir = a
if o in ('--release', ):
rel = a
if o in ('--tag', ):
tag = a
if o in ('--dryRun', ):
dryRun = True
if o in ('--cycle', ):
cycle = a
config.setDefaults(cycle)
cfg = config.Configuration[cycle]
rb = ReleaseBuilder(buildDir, cfg)
if dryRun: rb.setDryRun()
try:
rb.doBuild(rel, tag)
except Exception, e:
print "ERROR: Caught exception during doBuild : " + str(e)
def main():
import getopt
options = sys.argv[1:]
try:
opts, args = getopt.getopt(options, "h", ["help", "buildDir=", "release=", "tag=", "dryRun", "cycle="])
except getopt.GetoptError:
usage()
sys.exit(-2)
buildDir = None
rel = None
tag = None
dryRun = False
cycle = None
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("--buildDir",):
buildDir = a
if o in ("--release",):
rel = a
if o in ("--tag",):
tag = a
if o in ("--dryRun",):
dryRun = True
if o in ("--cycle",):
cycle = a
config.setDefaults(cycle)
cfg = config.Configuration[cycle]
rb = ReleaseBuilder(buildDir, cfg)
if dryRun:
rb.setDryRun()
try:
rb.doBuild(rel, tag)
except Exception, e:
print "ERROR: Caught exception during doBuild : " + str(e)
def __init__(self,
dayIn=None,
relCycIn=None,
dryRunIn=False,
doInstall=True):
BuilderBase.__init__(self)
self.day = dayIn
if not self.day:
self.day = time.strftime("%a").lower()
self.dryRun = dryRunIn
self.topDir = os.path.join(self.installDir, self.plat)
self.relCycle = relCycIn
self.doInstall = doInstall
self.cand = None
config.setDefaults(relCycIn)
return
def main():
parser = OptionParser()
usage = "usage: %prog [option1] arg1 [option2] arg2 ... [optionN] argN"
parser = OptionParser(usage=usage)
parser.add_option("-b", "--buildDir",
dest="buildDir",
help="build directory")
parser.add_option("-r", "--releaseDir",
dest="releaseDir",
help="release directory")
parser.add_option("-d", "--dryRun",
action="store_true", dest="dryRun",
help="is a dry run?")
parser.add_option("-p", "--platform",
dest="platform",
help="used architecture")
parser.add_option("-c", "--relCycle",
dest="relCycle",
help="Release cycle")
parser.add_option("-s", "--stamp",
dest="stamp",
help="Release stamp")
parser.add_option("-t", "--relTag",
dest="relTag",
help="Release tag")
parser.add_option("-u", "--custom_ib",
action="store_true",
dest="custom_ib",
help="Sets the specific configuration for running DQM comparison for Custom IBs.")
parser.add_option("-e", "--report_path",
dest="report_path",
help="Path to report. Optional value")
parser.add_option("-o", "--report_relative_path",
dest="report_relative_path",
help="Relative Path to report. The value is used for compiling report web page url. Optional value")
(options, args) = parser.parse_args()
dryRun = False
if not options.dryRun == None:
dryRun = options.dryRun
custom_ib = False
if not options.custom_ib == None:
custom_ib = options.custom_ib
if options.buildDir == None:
print "The Integration Build directory is not specified"
return errno.EINVAL
buildDir = options.buildDir
report_path = options.report_path
report_relative_path = options.report_relative_path
while buildDir.endswith('/'):
buildDir = buildDir[:(len(buildDir)-1)]
(head, stamp) = os.path.split(buildDir)
head.strip() #hack, don't show never read warning
if options.stamp == None:
stamp = str(stamp).strip()
else:
stamp = options.stamp
if options.relTag == None:
buildName = getBuildName(buildDir)
else:
buildName = options.relTag
if options.platform == None:
buildArchitecture = getBuildArchitecture(buildName, buildDir)
else:
buildArchitecture = options.platform
if options.relCycle == None:
rel = getRelVersion(buildName)
else:
rel = options.relCycle
intBldFQNPath = os.path.dirname(os.path.dirname(buildDir))
if not options.releaseDir == None:
buildNameFQNPath = options.releaseDir
else:
buildNameFQNPath = os.path.join(buildDir, buildName)
if not os.path.exists(buildNameFQNPath):
print 'Build is not found in common path %s' % str(buildNameFQNPath)
buildNameFQNPath = os.path.join(intBldFQNPath, 'cms', buildArchitecture, 'cms', 'cmssw', buildName)
print 'Looking for the build in %s' % str(buildNameFQNPath)
if not os.path.exists(buildNameFQNPath):
print 'The build location cannot be found, terminating ...'
return errno.ENOENT
import config
config.setDefaults(rel)
relMonParameters = config.Configuration[rel]['RelMonParameters']
if custom_ib:
deployer_spec_file = os.path.join( buildNameFQNPath, 'tmp', 'tmpspec-cmssw-dqm-reference-deployer')
referenceBuildName = config.getDQMReferenceBuild(deployer_spec_file)
else:
referenceBuildName = config.getDQMReferenceBuild()
if not referenceBuildName:
print "Reference build doesn't exist in configuration, exiting comparison ..."
return errno.ENOENT
referenceBuildFQNPath = getReferenceBuildFQNPath(config, rel, referenceBuildName, buildArchitecture);
if not os.path.exists(referenceBuildFQNPath):
referenceBuildFQNPath2 = os.path.join(buildNameFQNPath, buildArchitecture, 'cms', 'cmssw-dqm-reference-deployer', referenceBuildName, 'data')
print "Reference build path %s doesn't exist, will search in %s" % (referenceBuildFQNPath,referenceBuildFQNPath2,)
referenceBuildFQNPath = referenceBuildFQNPath2
(referenceBuildName, referenceBuildArchitecture) = searchReferenceBuildNameAndArchitectureInPath(referenceBuildFQNPath)
threshold = getRelMonValue(relMonParameters,'threshold')
statTest = getRelMonValue(relMonParameters,'statTest')
doPngs = getRelMonValue(relMonParameters,'doPngs')
doComparison = getRelMonValue(relMonParameters,'doComparison')
doReport = getRelMonValue(relMonParameters,'doReport')
no_successes = getRelMonValue(relMonParameters,'no_successes')
black_list = getRelMonValue(relMonParameters,'black_list')
success_percentage = getRelMonValue(relMonParameters, 'success_percentage')
workflowFQNParentPath = os.path.join(buildNameFQNPath, workflowDataRelativePath)
if not os.path.exists(workflowFQNParentPath):
workflowFQNParentPath2 = os.path.join(buildNameFQNPath, workflowDataRelativePath_tests)
print 'The workflow location %s cannot be found, will search in %s' % (workflowFQNParentPath, workflowFQNParentPath2,)
workflowFQNParentPath = workflowFQNParentPath2
if not os.path.exists(workflowFQNParentPath):
print 'The workflow location %s cannot be found, terminating ...' % workflowFQNParentPath
return errno.ENOENT
dirs = os.listdir(workflowFQNParentPath)
wThreads = {}
for workflowDir in dirs:
if 'HARVEST' in workflowDir:
workflow = getWorkflowNumber(workflowDir)
workflowDirFQN = os.path.join(workflowFQNParentPath, workflowDir)
dqmHarvestedRootFile = getDQMHarvestedRootFile(workflowDirFQN)
if dqmHarvestedRootFile is not None:
rootfileBuildFQN = os.path.join(workflowDirFQN, dqmHarvestedRootFile)
## reference build root file
referenceWorkflows = os.listdir(referenceBuildFQNPath)
referenceBuildWorkflowFQNPath = None
for referenceWorkflow in referenceWorkflows:
if referenceWorkflow.startswith(str(workflow)):
referenceBuildWorkflowFQNPath = os.path.join(referenceBuildFQNPath, referenceWorkflow)
break
if referenceBuildWorkflowFQNPath == None:
print "Workflow ", str(workflow), " doesn't exist in the reference build folder ", str(referenceBuildFQNPath)
continue #go to next workflow iw current one is missing
refRootfileBuild = getDQMHarvestedRootFile(referenceBuildWorkflowFQNPath)
if refRootfileBuild is not None:
refRootfileBuildFQN = os.path.join(referenceBuildWorkflowFQNPath, refRootfileBuild)
print "comparing file ", rootfileBuildFQN, " with ", refRootfileBuildFQN
outputReportName = workflow
outputReportDir = os.path.join(buildNameFQNPath, outputReportBase, outputReportName)
increment = 0
while os.path.exists(outputReportDir):
increment = increment + 1
outputReportDir = outputReportDir + '-' + str(increment)
os.makedirs(outputReportDir)
waitThreads(wThreads,config.MachineCPUCount)
thrd = Thread(target=makeComparison, args=(buildNameFQNPath,
buildArchitecture,
no_successes,
rootfileBuildFQN,
refRootfileBuildFQN,
outputReportDir,
threshold,
statTest,
doPngs,
doComparison,
doReport,
workflowDir,
workflowFQNParentPath,
black_list,
success_percentage,
stamp,
rel,
outputReportName,
buildName,
workflow,
referenceBuildName,
report_path,
report_relative_path,
dryRun))
wThreads[thrd]=1
thrd.start()
time.sleep(2)
else:
print '%s folder doesn\'t contain the referencing DQM harvested root file.' % workflowDir
else:
print '%s folder doesn\'t contain the DQM harvested root file.' % workflowDir
waitThreads(wThreads)
#if os.path.exists(outputReportDir):
# print 'Deleting temporary build report dir output'
# doCmd.doCmd('rm -rf %s' % outputReportDir, dryRun)
print "Done"
return
def main():
parser = OptionParser()
usage = "usage: %prog [option1] arg1 [option2] arg2 ... [optionN] argN"
parser = OptionParser(usage=usage)
parser.add_option("-b",
"--buildDir",
dest="buildDir",
help="build directory")
parser.add_option("-r",
"--releaseDir",
dest="releaseDir",
help="release directory")
parser.add_option("-d",
"--dryRun",
action="store_true",
dest="dryRun",
help="is a dry run?")
parser.add_option("-p",
"--platform",
dest="platform",
help="used architecture")
parser.add_option("-c",
"--relCycle",
dest="relCycle",
help="Release cycle")
parser.add_option("-s", "--stamp", dest="stamp", help="Release stamp")
parser.add_option("-t", "--relTag", dest="relTag", help="Release tag")
parser.add_option(
"-u",
"--custom_ib",
action="store_true",
dest="custom_ib",
help=
"Sets the specific configuration for running DQM comparison for Custom IBs."
)
parser.add_option("-e",
"--report_path",
dest="report_path",
help="Path to report. Optional value")
parser.add_option(
"-o",
"--report_relative_path",
dest="report_relative_path",
help=
"Relative Path to report. The value is used for compiling report web page url. Optional value"
)
(options, args) = parser.parse_args()
dryRun = False
if not options.dryRun == None:
dryRun = options.dryRun
custom_ib = False
if not options.custom_ib == None:
custom_ib = options.custom_ib
if options.buildDir == None:
print "The Integration Build directory is not specified"
return errno.EINVAL
buildDir = options.buildDir
report_path = options.report_path
report_relative_path = options.report_relative_path
while buildDir.endswith('/'):
buildDir = buildDir[:(len(buildDir) - 1)]
(head, stamp) = os.path.split(buildDir)
head.strip() #hack, don't show never read warning
if options.stamp == None:
stamp = str(stamp).strip()
else:
stamp = options.stamp
if options.relTag == None:
buildName = getBuildName(buildDir)
else:
buildName = options.relTag
if options.platform == None:
buildArchitecture = getBuildArchitecture(buildName, buildDir)
else:
buildArchitecture = options.platform
if options.relCycle == None:
rel = getRelVersion(buildName)
else:
rel = options.relCycle
intBldFQNPath = os.path.dirname(os.path.dirname(buildDir))
if not options.releaseDir == None:
buildNameFQNPath = options.releaseDir
else:
buildNameFQNPath = os.path.join(buildDir, buildName)
if not os.path.exists(buildNameFQNPath):
print 'Build is not found in common path %s' % str(buildNameFQNPath)
buildNameFQNPath = os.path.join(intBldFQNPath, 'cms',
buildArchitecture, 'cms', 'cmssw',
buildName)
print 'Looking for the build in %s' % str(buildNameFQNPath)
if not os.path.exists(buildNameFQNPath):
print 'The build location cannot be found, terminating ...'
return errno.ENOENT
import config
config.setDefaults(rel)
relMonParameters = config.Configuration[rel]['RelMonParameters']
if custom_ib:
deployer_spec_file = os.path.join(
buildNameFQNPath, 'tmp', 'tmpspec-cmssw-dqm-reference-deployer')
referenceBuildName = config.getDQMReferenceBuild(deployer_spec_file)
else:
referenceBuildName = config.getDQMReferenceBuild()
if not referenceBuildName:
print "Reference build doesn't exist in configuration, exiting comparison ..."
return errno.ENOENT
referenceBuildFQNPath = getReferenceBuildFQNPath(config, rel,
referenceBuildName,
buildArchitecture)
if not os.path.exists(referenceBuildFQNPath):
referenceBuildFQNPath2 = os.path.join(buildNameFQNPath,
buildArchitecture, 'cms',
'cmssw-dqm-reference-deployer',
referenceBuildName, 'data')
print "Reference build path %s doesn't exist, will search in %s" % (
referenceBuildFQNPath,
referenceBuildFQNPath2,
)
referenceBuildFQNPath = referenceBuildFQNPath2
(referenceBuildName, referenceBuildArchitecture
) = searchReferenceBuildNameAndArchitectureInPath(referenceBuildFQNPath)
threshold = getRelMonValue(relMonParameters, 'threshold')
statTest = getRelMonValue(relMonParameters, 'statTest')
doPngs = getRelMonValue(relMonParameters, 'doPngs')
doComparison = getRelMonValue(relMonParameters, 'doComparison')
doReport = getRelMonValue(relMonParameters, 'doReport')
no_successes = getRelMonValue(relMonParameters, 'no_successes')
black_list = getRelMonValue(relMonParameters, 'black_list')
success_percentage = getRelMonValue(relMonParameters, 'success_percentage')
workflowFQNParentPath = os.path.join(buildNameFQNPath,
workflowDataRelativePath)
if not os.path.exists(workflowFQNParentPath):
workflowFQNParentPath2 = os.path.join(buildNameFQNPath,
workflowDataRelativePath_tests)
print 'The workflow location %s cannot be found, will search in %s' % (
workflowFQNParentPath,
workflowFQNParentPath2,
)
workflowFQNParentPath = workflowFQNParentPath2
if not os.path.exists(workflowFQNParentPath):
print 'The workflow location %s cannot be found, terminating ...' % workflowFQNParentPath
return errno.ENOENT
dirs = os.listdir(workflowFQNParentPath)
wThreads = {}
for workflowDir in dirs:
if 'HARVEST' in workflowDir:
workflow = getWorkflowNumber(workflowDir)
workflowDirFQN = os.path.join(workflowFQNParentPath, workflowDir)
dqmHarvestedRootFile = getDQMHarvestedRootFile(workflowDirFQN)
if dqmHarvestedRootFile is not None:
rootfileBuildFQN = os.path.join(workflowDirFQN,
dqmHarvestedRootFile)
## reference build root file
referenceWorkflows = os.listdir(referenceBuildFQNPath)
referenceBuildWorkflowFQNPath = None
for referenceWorkflow in referenceWorkflows:
if referenceWorkflow.startswith(str(workflow)):
referenceBuildWorkflowFQNPath = os.path.join(
referenceBuildFQNPath, referenceWorkflow)
break
if referenceBuildWorkflowFQNPath == None:
print "Workflow ", str(
workflow
), " doesn't exist in the reference build folder ", str(
referenceBuildFQNPath)
continue #go to next workflow iw current one is missing
refRootfileBuild = getDQMHarvestedRootFile(
referenceBuildWorkflowFQNPath)
if refRootfileBuild is not None:
refRootfileBuildFQN = os.path.join(
referenceBuildWorkflowFQNPath, refRootfileBuild)
print "comparing file ", rootfileBuildFQN, " with ", refRootfileBuildFQN
outputReportName = workflow
outputReportDir = os.path.join(buildNameFQNPath,
outputReportBase,
outputReportName)
increment = 0
while os.path.exists(outputReportDir):
increment = increment + 1
outputReportDir = outputReportDir + '-' + str(
increment)
os.makedirs(outputReportDir)
waitThreads(wThreads, config.MachineCPUCount)
thrd = Thread(
target=makeComparison,
args=(buildNameFQNPath, buildArchitecture,
no_successes, rootfileBuildFQN,
refRootfileBuildFQN, outputReportDir, threshold,
statTest, doPngs, doComparison, doReport,
workflowDir, workflowFQNParentPath, black_list,
success_percentage, stamp, rel, outputReportName,
buildName, workflow, referenceBuildName,
report_path, report_relative_path, dryRun))
wThreads[thrd] = 1
thrd.start()
time.sleep(2)
else:
print '%s folder doesn\'t contain the referencing DQM harvested root file.' % workflowDir
else:
print '%s folder doesn\'t contain the DQM harvested root file.' % workflowDir
waitThreads(wThreads)
#if os.path.exists(outputReportDir):
# print 'Deleting temporary build report dir output'
# doCmd.doCmd('rm -rf %s' % outputReportDir, dryRun)
print "Done"
return