def run(self):
IBThreadBase.run(self)
from runPyRelValThread import PyRelValsThread
pyrelval = PyRelValsThread(8, self.startDir+'/pyRelval')
try:
add_arg = re.sub('"\s*$',"",re.sub('^\s*"',"",self.cmd.split("--args ")[1]))
runCmd('cd '+self.startDir+'; rm -rf pyRelval; mkdir pyRelval')
pyrelval.startWorkflows(self.logger, add_arg)
except Exception, e :
print "runTests> ERROR during test PyReleaseValidation : caught exception: " + str(e)
pass
if __name__ == "__main__":
parser = OptionParser(usage="%prog -i|--id <jobid> -l|--list <list of workflows>")
parser.add_option("-i", "--id", dest="jobid", help="Job Id e.g. 1of3", default="1of1")
parser.add_option("-l", "--list", dest="workflow", help="List of workflows to run e.g. 1.0,2.0,3.0", type=str, default=None)
parser.add_option("-f", "--force",dest="force", help="Force running of workflows without checking the server for previous run", action="store_true", default=False)
parser.add_option("-d", "--das-cache", dest="das_cache", help="Das cache file", type=str, default=None)
opts, args = parser.parse_args()
if len(args) > 0: parser.error("Too many/few arguments")
if not opts.workflow: parser.error("Missing -l|--list <workflows> argument.")
if (not environ.has_key("CMSSW_VERSION")) or (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
exit(1)
thrds = cmsRunProcessCount
if isThreaded(environ["CMSSW_VERSION"],environ["SCRAM_ARCH"]):
print "Treaded IB Found"
thrds=int(MachineMemoryGB/5)
elif "fc22_" in environ["SCRAM_ARCH"]:
print "FC22 IB Found"
thrds=int(MachineMemoryGB/4)
else:
print "Normal IB Found"
if thrds>cmsRunProcessCount: thrds=cmsRunProcessCount
matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"]+"/pyRelval", opts.jobid)
matrix.setArgs(GetMatrixOptions(environ["CMSSW_VERSION"],environ["SCRAM_ARCH"], opts.das_cache))
matrix.run_workflows(opts.workflow.split(","),LogUpdater(environ["CMSSW_BASE"]),opts.force)
from os.path import exists
from RelValArgs import GetMatrixOptions
from runPyRelValThread import PyRelValsThread, splitWorkflows
from cmsutils import doCmd
print "Diving workflows"
workdir = sys.argv[1]
RelValtimes = sys.argv[2]
print RelValtimes
try:
max_wf = int(sys.argv[3])
except:
max_wf = 100
relval_args = GetMatrixOptions(environ["CMSSW_VERSION"], environ["SCRAM_ARCH"])
if 'RELVAL_WORKFLOWS' in environ:
relval_args = relval_args + ' ' + environ["RELVAL_WORKFLOWS"]
matrix = PyRelValsThread(1, environ["CMSSW_BASE"])
workflows = matrix.getWorkFlows(relval_args)
if exists(RelValtimes):
owf = []
max_tm = 0
with open(RelValtimes) as json_file:
try:
json_data = json.load(json_file)
except:
print "Error reading RelVal Times"
json_data = {"avg": []}
for tm_str in sorted(json_data["avg"], key=int, reverse=True):
tm = int(tm_str)
if tm > max_tm: max_tm = tm
for wf in json_data["avg"][tm_str]:
if wf in workflows: owf.append([wf, tm])
shell=True)
e = waitpid(p.pid, 0)[1]
print("Time took to create jobs:", int(time() - stime), "sec")
system("touch " + cmssw_base + "/done." + opts.jobid)
if logger:
logger.updateRelValMatrixPartialLogs(cmssw_base,
"done." + opts.jobid)
exit(e)
if isThreaded(cmssw_ver, arch):
print("Threaded IB Found")
thrds = int(MachineMemoryGB / 4.5)
if thrds == 0: thrds = 1
elif "fc24_ppc64le_" in arch:
print("FC22 IB Found")
thrds = int(MachineMemoryGB / 4)
elif "fc24_ppc64le_" in arch:
print("CentOS 7.2 + PPC64LE Found")
thrds = int(MachineMemoryGB / 3)
else:
print("Normal IB Found")
if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount
known_errs = get_known_errors(cmssw_ver, arch, "relvals")
matrix = PyRelValsThread(thrds, cmssw_base + "/pyRelval", opts.jobid)
matrix.setArgs(GetMatrixOptions(cmssw_ver, arch))
matrix.run_workflows(opts.workflow.split(","),
logger,
opts.force,
known_errors=known_errs)
arch = environ["SCRAM_ARCH"]
if isThreaded(cmssw_ver,arch):
print "Threaded IB Found"
thrds=int(MachineMemoryGB/4.5)
if thrds==0: thrds=1
elif "fc24_ppc64le_" in arch:
print "FC22 IB Found"
thrds=int(MachineMemoryGB/4)
elif "fc24_ppc64le_" in arch:
print "CentOS 7.2 + PPC64LE Found"
thrds=int(MachineMemoryGB/3)
else:
print "Normal IB Found"
if thrds>cmsRunProcessCount: thrds=cmsRunProcessCount
known_errs = get_known_errors(cmssw_ver, arch, "relvals")
matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"]+"/pyRelval", opts.jobid)
matrix.setArgs(GetMatrixOptions(cmssw_ver,arch))
#print matrix.args
#print GetMatrixOptions(cmssw_ver,arch)
#print matrix.args['rest']
wfs = opts.workflow.split(",")
for wf in wfs:
print 'flow is :', wf
#matrix.args['rest'] = FixWFArgs(cmssw_ver,arch,wf,matrix.args['rest']).replace(' -t 4','')
#print fixed_args
#matrix.args['rest'] = fixed_args.replace(' -t 4','')
#print matrix.args['rest']
#print matrix.args['w']
runStep1Only(matrix.basedir, wf, matrix.args['rest']+" "+matrix.args['w'])
wf_folder = [f for f in os.listdir(matrix.basedir) if f.find(wf+'_') is not -1][0]
from os import environ
from os.path import exists
from RelValArgs import GetMatrixOptions
from runPyRelValThread import PyRelValsThread, splitWorkflows
from cmsutils import doCmd
print("Diving workflows")
workdir = sys.argv[1]
RelValtimes = sys.argv[2]
print(RelValtimes)
try:
max_wf=int(sys.argv[3])
except:
max_wf=100
relval_args = GetMatrixOptions(environ["CMSSW_VERSION"], environ["SCRAM_ARCH"])
if 'RELVAL_WORKFLOWS' in environ: relval_args=relval_args+' '+environ["RELVAL_WORKFLOWS"]
matrix = PyRelValsThread(1,environ["CMSSW_BASE"])
workflows = matrix.getWorkFlows(relval_args)
if exists(RelValtimes):
owf = []
max_tm=0
with open(RelValtimes) as json_file:
try: json_data = json.load(json_file)
except:
print("Error reading RelVal Times")
json_data={"avg": []}
for tm_str in sorted(json_data["avg"],key=int, reverse=True):
tm=int(tm_str)
if tm > max_tm : max_tm=tm
for wf in json_data["avg"][tm_str]:
if wf in workflows: owf.append([wf,tm])
uwf = []
system("touch "+cmssw_base+"/done."+opts.jobid)
if logger: logger.updateRelValMatrixPartialLogs(cmssw_base, "done."+opts.jobid)
exit(e)
if re.match("^CMSSW_(9|[1-9][0-9]+)_([3-9]|[1-9][0-9]+)_.+$",cmssw_ver):
p=Popen("%s/jobs/create-relval-jobs.py %s" % (SCRIPT_DIR, opts.workflow),shell=True)
e=waitpid(p.pid,0)[1]
if e: exit(e)
p = Popen("cd %s/pyRelval ; %s/jobs/jobscheduler.py -M 0 -c 175 -m 90 -o dynamic -t avg" % (cmssw_base,SCRIPT_DIR), shell=True)
e=waitpid(p.pid,0)[1]
system("touch "+cmssw_base+"/done."+opts.jobid)
if logger: logger.updateRelValMatrixPartialLogs(cmssw_base, "done."+opts.jobid)
exit(e)
if isThreaded(cmssw_ver,arch):
print "Threaded IB Found"
thrds=int(MachineMemoryGB/4.5)
if thrds==0: thrds=1
elif "fc24_ppc64le_" in arch:
print "FC22 IB Found"
thrds=int(MachineMemoryGB/4)
elif "fc24_ppc64le_" in arch:
print "CentOS 7.2 + PPC64LE Found"
thrds=int(MachineMemoryGB/3)
else:
print "Normal IB Found"
if thrds>cmsRunProcessCount: thrds=cmsRunProcessCount
known_errs = get_known_errors(cmssw_ver, arch, "relvals")
matrix = PyRelValsThread(thrds, cmssw_base+"/pyRelval", opts.jobid)
matrix.setArgs(GetMatrixOptions(cmssw_ver,arch))
matrix.run_workflows(opts.workflow.split(","),logger,opts.force,known_errors=known_errs)
arch = environ["SCRAM_ARCH"]
if isThreaded(cmssw_ver, arch):
print "Threaded IB Found"
thrds = int(MachineMemoryGB / 4.5)
if thrds == 0: thrds = 1
elif "fc24_ppc64le_" in arch:
print "FC22 IB Found"
thrds = int(MachineMemoryGB / 4)
elif "fc24_ppc64le_" in arch:
print "CentOS 7.2 + PPC64LE Found"
thrds = int(MachineMemoryGB / 3)
else:
print "Normal IB Found"
if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount
known_errs = get_known_errors(cmssw_ver, arch, "relvals")
matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"] + "/pyRelval",
opts.jobid)
matrix.setArgs(GetMatrixOptions(cmssw_ver, arch))
#print matrix.args
#print GetMatrixOptions(cmssw_ver,arch)
#print matrix.args['rest']
wfs = opts.workflow.split(",")
for wf in wfs:
print 'flow is :', wf
#matrix.args['rest'] = FixWFArgs(cmssw_ver,arch,wf,matrix.args['rest']).replace(' -t 4','')
#print fixed_args
#matrix.args['rest'] = fixed_args.replace(' -t 4','')
#print matrix.args['rest']
#print matrix.args['w']
runStep1Only(matrix.basedir, wf,
matrix.args['rest'] + " " + matrix.args['w'])
#!/usr/bin/env python
from __future__ import print_function
import os, sys
from runPyRelValThread import PyRelValsThread
path=sys.argv[1]
newloc = os.path.dirname(path) + '/pyRelValMatrixLogs/run'
os.system('mkdir -p ' + newloc)
ProcessLogs = PyRelValsThread(1,path,"1of1",newloc)
print("Generating runall log file: %s" % path)
ProcessLogs.update_runall()
print("Generating relval time info")
ProcessLogs.update_wftime()
print("Parsing logs for workflows/steps")
ProcessLogs.parseLog()
print("Done")
if (not environ.has_key("CMSSW_VERSION")) or (
not environ.has_key("CMSSW_BASE")) or (
not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
exit(1)
thrds = cmsRunProcessCount
cmssw_ver = environ["CMSSW_VERSION"]
arch = environ["SCRAM_ARCH"]
if isThreaded(cmssw_ver, arch):
print "Threaded IB Found"
thrds = int(MachineMemoryGB / 4.5)
if thrds == 0: thrds = 1
elif "fc24_ppc64le_" in arch:
print "FC22 IB Found"
thrds = int(MachineMemoryGB / 4)
elif "fc24_ppc64le_" in arch:
print "CentOS 7.2 + PPC64LE Found"
thrds = int(MachineMemoryGB / 3)
else:
print "Normal IB Found"
if thrds > cmsRunProcessCount: thrds = cmsRunProcessCount
known_errs = get_known_errors(cmssw_ver, arch, "relvals")
matrix = PyRelValsThread(thrds, environ["CMSSW_BASE"] + "/pyRelval",
opts.jobid)
matrix.setArgs(GetMatrixOptions(cmssw_ver, arch))
matrix.run_workflows(opts.workflow.split(","),
LogUpdater(environ["CMSSW_BASE"]),
opts.force,
known_errors=known_errs)
#!/usr/bin/env python
import os, sys
from runPyRelValThread import PyRelValsThread
path=sys.argv[1]
newloc = os.path.dirname(path) + '/pyRelValMatrixLogs/run'
os.system('mkdir -p ' + newloc)
ProcessLogs = PyRelValsThread(1,path,"1of1",newloc)
print "Generating runall log file"
ProcessLogs.update_runall()
print "Generating relval time info"
ProcessLogs.update_wftime()
print "Parsing logs for workflows/steps"
ProcessLogs.parseLog()
print "Done"