def setupLogger(self, doInstall=True):
if not self.logger:
from logUpdater import LogUpdater
self.logger = LogUpdater(self.buildDir, doInstall)
self.logger.setDryRun(self.dryRun)
self.logger.setRelease(self.relTag, self.relCycle, self.stamp)
return
def startBuild(self, buildDir=None):
if buildDir: self.buildDir = buildDir
os.environ['CMSINTBLD_RCDIR'] = self.buildDir
self.updateTimeStamp(self.buildDir)
os.environ[
"PATH"] = self.buildDir + "/cms/common:" + os.environ["PATH"]
from logUpdater import LogUpdater
logger = LogUpdater(self.buildDir)
logger.setDryRun(self.dryRun)
logger.setRelease("", self.relCycle, self.stamp)
if not os.path.exists(self.cmsBuildDir):
os.makedirs(self.cmsBuildDir)
logDir = self.buildDir + '/' + self.relCycle + '/' + self.plat
logFileName = logDir + "/fullLog"
cmd = 'rm -rf ' + self.relCycle + '; mkdir -p ' + logDir + '; '
cmd += scriptPath + "/buildNightly.py "
cmd += " --releasecycle " + self.relCycle + ' '
cmd += " --buildDir " + self.buildDir + ' '
if self.dryRun: cmd += ' --dryRun '
cmd += ' > ' + logFileName + ' 2>&1 '
self.doCmd(cmd, True, self.buildDir)
print "log file at " + self.buildDir + "/fullLog"
webLogPath = logger.webTargetDir
if not os.path.exists(webLogPath):
os.makedirs(webLogPath)
cmd = 'cp ' + logFileName + ' ' + webLogPath
self.doCmd(cmd, True)
if self.dryRun:
sys.stdout.flush()
sys.stderr.flush()
os.system('cat ' + logFileName)
else:
print "\n"
print "web portal at:\n"
print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py?rc=" + self.relCycle + "\n"
print "http://cmssdt.cern.ch/SDT/cgi-bin/showIB.py\n"
print "+" * 80
print "\n"
sys.stdout.flush()
sys.stderr.flush()
return
def __init__(self, releaseDir, dryRun=False):
self.dryRun = dryRun
self.plat = os.environ["SCRAM_ARCH"]
self.appset = releaseDir + "/CMSDIST"
self.cmsswBuildDir = releaseDir
self.release = os.path.basename(releaseDir)
self.relTag = self.release
self.threadList = {}
from cmsutils import getIBReleaseInfo
self.relCycle, day, hour = getIBReleaseInfo(self.release)
from logUpdater import LogUpdater
self.logger = LogUpdater(self.cmsswBuildDir, self.dryRun)
return
def upload_logs(workflow, workflow_dir):
basedir = os.path.dirname(workflow_dir)
for wf_file in glob.glob("%s/*" % workflow_dir):
found=False
for ext in [ ".txt", ".xml", ".log", ".py", ".json","/cmdLog", "/hostname",".done" ]:
if wf_file.endswith(ext):
found=True
break
if not found:
print "Removing ",wf_file
getstatusoutput("rm -rf %s" % wf_file)
logger=LogUpdater(dirIn=os.environ["CMSSW_BASE"])
logger.updateRelValMatrixPartialLogs(basedir, os.path.basename(workflow_dir))
def upload_logs(workflow, workflow_dir,exit_code):
files_to_keep = [ ".txt", ".xml", ".log", ".py", ".json","/cmdLog", "/hostname",".done" ]
if (exit_code in [34304, 35584, 22016]) and os.getenv("CMSSW_VERSION","").startswith("CMSSW_10_1_"):
files_to_keep.append(".root")
basedir = os.path.dirname(workflow_dir)
for wf_file in glob.glob("%s/*" % workflow_dir):
found=False
for ext in files_to_keep:
if wf_file.endswith(ext):
found=True
break
if not found:
print "Removing ",wf_file
getstatusoutput("rm -rf %s" % wf_file)
logger=LogUpdater(dirIn=os.environ["CMSSW_BASE"])
logger.updateRelValMatrixPartialLogs(basedir, os.path.basename(workflow_dir))
if __name__ == "__main__":
parser = OptionParser(usage="%prog -i|--id <jobid> -l|--list <list of workflows>")
parser.add_option("-i", "--id", dest="jobid", help="Job Id e.g. 1of3", default="1of1")
parser.add_option("-l", "--list", dest="workflow", help="List of workflows to run e.g. 1.0,2.0,3.0", type=str, default=None)
parser.add_option("-f", "--force",dest="force", help="Force running of workflows without checking the server for previous run", action="store_true", default=False)
parser.add_option("-d", "--das-cache", dest="das_cache", help="Das cache file", type=str, default=None)
opts, args = parser.parse_args()
if len(args) > 0: parser.error("Too many/few arguments")
if not opts.workflow: parser.error("Missing -l|--list <workflows> argument.")
if (not environ.has_key("CMSSW_VERSION")) or (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
exit(1)
thrds = cmsRunProcessCount
if isThreaded(environ["CMSSW_VERSION"],environ["SCRAM_ARCH"]):
print "Treaded IB Found"
thrds=int(MachineMemoryGB/5)
elif "fc22_" in environ["SCRAM_ARCH"]:
print "FC22 IB Found"
thrds=int(MachineMemoryGB/4)
else:
print "Normal IB Found"
if thrds>cmsRunProcessCount: thrds=cmsRunProcessCount
matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"]+"/pyRelval", opts.jobid)
matrix.setArgs(GetMatrixOptions(environ["CMSSW_VERSION"],environ["SCRAM_ARCH"], opts.das_cache))
matrix.run_workflows(opts.workflow.split(","),LogUpdater(environ["CMSSW_BASE"]),opts.force)
if ("CMSSW_VERSION" not in environ) or ("CMSSW_BASE" not in environ) or (
"SCRAM_ARCH" not in environ):
print(
"ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
)
exit(1)
if opts.dryRun: environ["CMSSW_DRY_RUN"] = "true"
if opts.nonThreaded: environ["CMSSW_NON_THREADED"] = "true"
elif "CMSSW_NON_THREADED" in environ: del os.environ['CMSSW_NON_THREADED']
thrds = cmsRunProcessCount
cmssw_ver = environ["CMSSW_VERSION"]
arch = environ["SCRAM_ARCH"]
cmssw_base = environ["CMSSW_BASE"]
logger = None
if not opts.dryRun: logger = LogUpdater(dirIn=cmssw_base)
if re.match("^CMSSW_(9_([3-9]|[1-9][0-9]+)|[1-9][0-9]+)_.*$", cmssw_ver):
stime = time()
p = Popen("%s/jobs/create-relval-jobs.py %s" %
(SCRIPT_DIR, opts.workflow),
shell=True)
e = waitpid(p.pid, 0)[1]
print("Time took to create jobs:", int(time() - stime), "sec")
if e: exit(e)
p = None
stime = time()
if cmssw_ver.find('_DUMMYCLANG_') is not -1:
cmssw_ver = cmssw_ver.rsplit('_', 1)[0] + '*'
p = Popen(
opts, args = parser.parse_args()
if len(args) > 0: parser.error("Too many/few arguments")
if not opts.workflow:
parser.error("Missing -l|--list <workflows> argument.")
if (not environ.has_key("CMSSW_VERSION")) or (
not environ.has_key("CMSSW_BASE")) or (
not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
exit(1)
thrds = cmsRunProcessCount
cmssw_ver = environ["CMSSW_VERSION"]
arch = environ["SCRAM_ARCH"]
cmssw_base = environ["CMSSW_BASE"]
logger = LogUpdater(dirIn=cmssw_base)
if re.match("^CMSSW_(9_([3-9]|[1-9][0-9]+)|[1-9][0-9]+)_.*$", cmssw_ver):
p = Popen("%s/jobs/create-relval-jobs.py %s" %
(SCRIPT_DIR, opts.workflow),
shell=True)
e = waitpid(p.pid, 0)[1]
if e: exit(e)
p = None
if cmssw_ver.find('_CLANG_') is not -1:
p = Popen(
"python %s/rv_scheduler/relval_main.py -a %s -r %s -d 7" %
(SCRIPT_DIR, arch, cmssw_ver.rsplit('_', 1)[0]),
shell=True)
else:
""":"
python_cmd="python3"
python -V >/dev/null 2>&1 && python_cmd="python"
exec ${python_cmd} $0 ${1+"$@"}
"""
from __future__ import print_function
from sys import exit, argv
from os import environ
from cmsutils import cmsRunProcessCount, doCmd
from logUpdater import LogUpdater
if ("CMSSW_BASE" not in environ) or ("SCRAM_ARCH" not in environ):
print(
"ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
)
exit(1)
timeout = 7200
try:
timeout = int(argv[1])
except:
timeout = 7200
logger = LogUpdater(environ["CMSSW_BASE"])
ret = doCmd(
'cd %s; rm -rf addOnTests; timeout %s addOnTests.py -j %s 2>&1 >addOnTests.log'
% (environ["CMSSW_BASE"], timeout, cmsRunProcessCount))
doCmd('cd ' + environ["CMSSW_BASE"] +
'/addOnTests/logs; zip -r addOnTests.zip *.log')
logger.updateAddOnTestsLogs()
"--das-cache",
dest="das_cache",
help="Das cache file",
type=str,
default=None)
opts, args = parser.parse_args()
if len(args) > 0: parser.error("Too many/few arguments")
if not opts.workflow:
parser.error("Missing -l|--list <workflows> argument.")
if (not environ.has_key("CMSSW_VERSION")) or (
not environ.has_key("CMSSW_BASE")) or (
not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
exit(1)
thrds = cmsRunProcessCount
if "_THREADED_" in environ["CMSSW_VERSION"]:
thrds = int(MachineMemoryGB / 5)
elif "fc22_" in environ["SCRAM_ARCH"]:
thrds = int(MachineMemoryGB / 4)
if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount
matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"] + "/pyRelval",
opts.jobid)
matrix.setArgs(
GetMatrixOptions(environ["CMSSW_VERSION"], environ["SCRAM_ARCH"],
opts.das_cache))
matrix.run_workflows(opts.workflow.split(","),
LogUpdater(environ["CMSSW_BASE"]), opts.force)
#!/usr/bin/env python from sys import argv from logUpdater import LogUpdater logger = LogUpdater(dirIn=argv[1]) logger.copyLogs(argv[2])
if (not environ.has_key("CMSSW_VERSION")) or (
not environ.has_key("CMSSW_BASE")) or (
not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
exit(1)
thrds = cmsRunProcessCount
cmssw_ver = environ["CMSSW_VERSION"]
arch = environ["SCRAM_ARCH"]
if isThreaded(cmssw_ver, arch):
print "Threaded IB Found"
thrds = int(MachineMemoryGB / 4.5)
if thrds == 0: thrds = 1
elif "fc24_ppc64le_" in arch:
print "FC22 IB Found"
thrds = int(MachineMemoryGB / 4)
elif "fc24_ppc64le_" in arch:
print "CentOS 7.2 + PPC64LE Found"
thrds = int(MachineMemoryGB / 3)
else:
print "Normal IB Found"
if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount
known_errs = get_known_errors(cmssw_ver, arch, "relvals")
matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"] + "/pyRelval",
opts.jobid)
matrix.setArgs(GetMatrixOptions(cmssw_ver, arch))
matrix.run_workflows(opts.workflow.split(","),
LogUpdater(environ["CMSSW_BASE"]),
opts.force,
known_errors=known_errs)
