def checkPyConfigs(self, deps = []):
print "Going to check python configs in ", os.getcwd()
cmd = scriptPath+'/checkPyConfigs.py > chkPyConf.log 2>&1'
try:
doCmd(cmd,self.dryRun,self.cmsswBuildDir)
self.logger.updateLogFile("chkPyConf.log")
self.logger.updateLogFile("chkPyConf.log",'testLogs')
except:
pass
return None
def checkPyConfigs(self, deps=[]):
print("Going to check python configs in ", os.getcwd())
cmd = scriptPath + '/checkPyConfigs.py > chkPyConf.log 2>&1'
try:
doCmd(cmd, self.dryRun, self.cmsswBuildDir)
self.logger.updateLogFile("chkPyConf.log")
self.logger.updateLogFile("chkPyConf.log", 'testLogs')
except:
pass
return None
def runDuplicateDictCheck(self, deps = []):
print "runDuplicateDictTests> Going to run duplicateReflexLibrarySearch.py ... "
script = 'duplicateReflexLibrarySearch.py'
for opt in ['dup', 'lostDefs', 'edmPD']:
cmd = script+' --'+opt+' 2>&1 >dupDict-'+opt+'.log'
try:
doCmd(cmd,self.dryRun,self.cmsswBuildDir)
except Exception, e :
print "ERROR during test duplicateDictCheck : caught exception: " + str(e)
self.logger.updateDupDictTestLogs()
def runProjectInit(self, deps = []):
print "runProjectInit> Going regenerate scram caches ... "
try:
ver=os.environ["CMSSW_VERSION"]
cmd= "cd "+self.cmsswBuildDir+"; rm -rf src;"
cmd+="curl -k -L -s -o src.tar.gz https://github.com/cms-sw/cmssw/archive/"+ver+".tar.gz;"
cmd+="tar -xzf src.tar.gz; mv cmssw-"+ver+" src; rm -rf src.tar.gz;"
cmd+="scram build -r echo_CXX"
doCmd(cmd)
except Exception, e :
print "ERROR during runProjectInit: caught exception: " + str(e)
def copy2RemoteHost(self, src, des, host):
cmd ="scp "+self.ssh_opt+" -r "+src+" "+host+":"+des
try:
if self.dryRun:
print "CMD>>",cmd
else:
for i in range (10):
err,out = doCmd(cmd)
if not err: return (err,out)
sleep(60)
return doCmd(cmd)
except Exception, e:
print "Ignoring exception during copy2Remote:", str(e)
return (1,str(e))
def copy2RemoteHost(self, src, des, host):
cmd = "scp " + self.ssh_opt + " -r " + src + " " + host + ":" + des
try:
if self.dryRun:
print("CMD>>", cmd)
else:
for i in range(10):
err, out = doCmd(cmd)
if not err: return (err, out)
sleep(60)
return doCmd(cmd)
except Exception as e:
print("Ignoring exception during copy2Remote:", str(e))
return (1, str(e))
def runRemoteHostCmd(self, cmd, host):
cmd ="ssh -Y "+self.ssh_opt+" "+host+" 'echo CONNECTION=OK && "+cmd+"'"
try:
if self.dryRun:
print "CMD>>",cmd
else:
for i in range (10):
err,out = doCmd(cmd)
if not err: return (err,out)
for l in out.split("\n"):
if "CONNECTION=OK" in l: return (err,out)
sleep(60)
return doCmd(cmd)
except Exception, e:
print "Ignoring exception during runRemoteCmd:", str(e)
return (1,str(e))
def runRemoteHostCmd(self, cmd, host):
cmd = "ssh -Y " + self.ssh_opt + " " + host + " 'echo CONNECTION=OK && " + cmd + "'"
try:
if self.dryRun:
print("CMD>>", cmd)
else:
for i in range(10):
err, out = doCmd(cmd)
if not err: return (err, out)
for l in out.split("\n"):
if "CONNECTION=OK" in l: return (err, out)
sleep(60)
return doCmd(cmd)
except Exception as e:
print("Ignoring exception during runRemoteCmd:", str(e))
return (1, str(e))
def run_workflows(self,
workflows=[],
logger=None,
force=False,
known_errors={}):
if not workflows: return
workflows = workflows[::-1]
threads = []
while (len(workflows) > 0):
threads = [t for t in threads if t.is_alive()]
if (len(threads) < self.jobs):
try:
wf = workflows.pop()
wf_err = {}
if wf in known_errors: wf_err = known_errors[wf]
t = threading.Thread(
target=runThreadMatrix,
args=(self.basedir, wf,
self.args['rest'] + " " + self.args['w'], logger,
force, wf_err))
t.start()
threads.append(t)
except Exception as e:
print(
"runPyRelVal> ERROR threading matrix : caught exception: "
+ str(e))
else:
time.sleep(5)
for t in threads:
t.join()
ret, out = doCmd("touch " + self.basedir + "/done." + self.jobid)
if logger:
logger.updateRelValMatrixPartialLogs(self.basedir,
"done." + self.jobid)
return
def getWorkFlows(self, args):
self.setArgs(args)
workflowsCmd = "runTheMatrix.py -n "+self.args['w']+" "+self.args['s']+" "+self.args['l']+" | grep -v ' workflows with ' | grep -E '^[0-9][0-9]*(\.[0-9][0-9]*|)\s\s*' | sort -nr | awk '{print $1}'"
print("RunTheMatrix>>",workflowsCmd)
cmsstat, workflows = doCmd(workflowsCmd)
if not cmsstat:
return workflows.split("\n")
print("runPyRelVal> ERROR during test PyReleaseValidation : could not get output of " + workflowsCmd)
return []
def getWorkFlows(self, args):
self.setArgs(args)
workflowsCmd = "runTheMatrix.py -n "+self.args['w']+" "+self.args['l']+" | grep -v ' workflows with ' | grep -E '^[0-9][0-9]*(\.[0-9][0-9]*|)\s\s*' | sort -nr | awk '{print $1}'"
print "RunTheMatrix>>",workflowsCmd
cmsstat, workflows = doCmd(workflowsCmd)
if not cmsstat:
return workflows.split("\n")
print "runPyRelVal> ERROR during test PyReleaseValidation : could not get output of " + workflowsCmd
return []
def runRemoteCmd(self, cmd):
cmd ="ssh -Y "+self.ssh_opt+" "+self.remote+" "+cmd
try:
if self.dryRun:
print "CMD>>",cmd
else:
return doCmd(cmd)
except Exception, e:
print "Ignoring exception during runRemoteCmd:", str(e)
return (1,str(e))
def copy2Remote(self, src, des):
cmd ="scp "+self.ssh_opt+" -r "+src+" "+self.remote+":"+des
try:
if self.dryRun:
print "CMD>>",cmd
else:
return doCmd(cmd)
except Exception, e:
print "Ignoring exception during copy2Remote:", str(e)
return (1,str(e))
def runRemoteCmd(self, cmd):
cmd = "ssh -Y " + self.ssh_opt + " " + self.remote + " " + cmd
try:
if self.dryRun:
print "CMD>>", cmd
else:
return doCmd(cmd)
except Exception, e:
print "Ignoring exception during runRemoteCmd:", str(e)
return (1, str(e))
def copy2Remote(self, src, des):
cmd = "scp " + self.ssh_opt + " -r " + src + " " + self.remote + ":" + des
try:
if self.dryRun:
print "CMD>>", cmd
else:
return doCmd(cmd)
except Exception, e:
print "Ignoring exception during copy2Remote:", str(e)
return (1, str(e))
class PyRelValsThread(object):
def __init__(self, jobs, basedir, jobid="1of1", outdir=None):
if not outdir: outdir = basedir
self.jobs = jobs
self.basedir = basedir
self.jobid = jobid
self.outdir = outdir
self.args = {}
self.setArgs("")
def setArgs(self, args):
args = args.replace('\\"', '"')
args, self.args['w'], tmp = find_argv(args, "-w|--what")
args, self.args['l'], tmp = find_argv(args, "-l|--list")
args, self.args['j'], tmp = find_argv(args, "-j|--nproc")
self.args['rest'] = args
def getWorkFlows(self, args):
self.setArgs(args)
workflowsCmd = "runTheMatrix.py -n " + self.args['w'] + " " + self.args[
'l'] + " | grep -v ' workflows with ' | grep -E '^[0-9][0-9]*(\.[0-9][0-9]*|)\s\s*' | sort -nr | awk '{print $1}'"
print "RunTheMatrix>>", workflowsCmd
cmsstat, workflows = doCmd(workflowsCmd)
if not cmsstat:
return workflows.split("\n")
print "runPyRelVal> ERROR during test PyReleaseValidation : could not get output of " + workflowsCmd
return []
def run_workflows(self, workflows=[], logger=None, force=False):
if not workflows: return
workflows = workflows[::-1]
threads = []
while (len(workflows) > 0):
threads = [t for t in threads if t.is_alive()]
if (len(threads) < self.jobs):
try:
t = threading.Thread(target=runThreadMatrix,
args=(self.basedir, workflows.pop(),
self.args['rest'] + " " +
self.args['w'], logger, force))
t.start()
threads.append(t)
except Exception, e:
print "runPyRelVal> ERROR threading matrix : caught exception: " + str(
e)
else:
time.sleep(5)
for t in threads:
t.join()
ret, out = doCmd("touch " + self.basedir + "/done." + self.jobid)
if logger:
logger.updateRelValMatrixPartialLogs(self.basedir,
"done." + self.jobid)
return
def runThreadMatrix(basedir,
workflow,
args='',
logger=None,
force=False,
wf_err={}):
if (not force) and logger and logger.relvalAlreadyDone(workflow):
print("Message>> Not runing workflow ", workflow,
" as it is already ran")
return
args = FixWFArgs(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"],
workflow, args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py -l ' + workflow + ' ' + args
try:
if not os.path.isdir(workdir):
os.makedirs(workdir)
except Exception as e:
print("runPyRelVal> ERROR during test PyReleaseValidation, workflow " +
str(workflow) + " : can't create thread folder: " + str(e))
wftime = time.time()
try:
ret = doCmd(matrixCmd, False, workdir)
except Exception as e:
print("runPyRelVal> ERROR during test PyReleaseValidation, workflow " +
str(workflow) + " : caught exception: " + str(e))
wftime = time.time() - wftime
outfolders = [
file for file in os.listdir(workdir)
if re.match("^" + str(workflow) + "_", file)
]
if len(outfolders) == 0: return
outfolder = os.path.join(basedir, outfolders[0])
wfdir = os.path.join(workdir, outfolders[0])
ret = doCmd("rm -rf " + outfolder + "; mkdir -p " + outfolder)
ret = doCmd(
"find . -mindepth 1 -maxdepth 1 -name '*.xml' -o -name '*.log' -o -name '*.py' -o -name '*.json' -o -name 'cmdLog' -type f | xargs -i mv '{}' "
+ outfolder + "/", False, wfdir)
logRE = re.compile('^(.*/[0-9]+(\.[0-9]+|)_([^/]+))/step1_dasquery.log$')
for logFile in glob.glob(outfolder + "/step1_dasquery.log"):
m = logRE.match(logFile)
if not m: continue
ret = doCmd("cp " + logFile + " " + m.group(1) + "/step1_" +
m.group(3) + ".log")
ret = doCmd("mv " + os.path.join(workdir, "runall-report-step*.log") +
" " + os.path.join(outfolder, "workflow.log"))
ret = doCmd("echo " + str(wftime) + " > " +
os.path.join(outfolder, "time.log"))
ret = doCmd("hostname -s > " + os.path.join(outfolder, "hostname"))
if wf_err: json.dump(wf_err, open("%s/known_error.json" % outfolder, "w"))
if logger: logger.updateRelValMatrixPartialLogs(basedir, outfolders[0])
shutil.rmtree(workdir)
return
def runStep1Only(basedir, workflow, args=''):
args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow +' '+args
try:
if not os.path.isdir(workdir):
os.makedirs(workdir)
except Exception as e:
print("runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : can't create thread folder: " + str(e))
try:
ret = doCmd(matrixCmd, False, workdir)
except Exception as e:
print("runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : caught exception: " + str(e))
return
from cmssw_known_errors import get_known_errors
from workflow_final import upload_logs
def runStep1Only(basedir, workflow, args=''):
args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow +' '+args
try:
if not os.path.isdir(basedir):
os.makedirs(basedir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : can't create thread folder: " + str(e)
try:
ret = doCmd(matrixCmd, False, basedir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : caught exception: " + str(e)
return
if __name__ == "__main__":
parser = OptionParser(usage="%prog -i|--id <jobid> -l|--list <list of workflows>")
parser.add_option("-i", "--id", dest="jobid", help="Job Id e.g. 1of3", default="1of1")
parser.add_option("-l", "--list", dest="workflow", help="List of workflows to run e.g. 1.0,2.0,3.0", type=str, default=None)
parser.add_option("-f", "--force",dest="force", help="Force running of workflows without checking the server for previous run", action="store_true", default=False)
opts, args = parser.parse_args()
if len(args) > 0: parser.error("Too many/few arguments")
if not opts.workflow: parser.error("Missing -l|--list <workflows> argument.")
if (not environ.has_key("CMSSW_VERSION")) or (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
def isNewRunTheMatrix(self):
e, o = doCmd("runTheMatrix.py --help | grep 'maxSteps=MAXSTEPS' | wc -l")
if e: return False
return o=="1"
from es_relval_log import es_parse_log
from RelValArgs import FixWFArgs
import json
from logreaderUtils import write_config_file, add_exception_to_config
def runStep1Only(basedir, workflow, args=''):
args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow +' '+args
try:
if not os.path.isdir(workdir):
os.makedirs(workdir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : can't create thread folder: " + str(e)
try:
ret = doCmd(matrixCmd, False, workdir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : caught exception: " + str(e)
return
def runThreadMatrix(basedir, workflow, args='', logger=None, force=False, wf_err={}):
if (not force) and logger and logger.relvalAlreadyDone(workflow):
print "Message>> Not runing workflow ",workflow," as it is already ran"
return
args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py -l ' + workflow +' '+args
try:
if not os.path.isdir(workdir):
os.makedirs(workdir)
except Exception, e:
def runStep1Only(basedir, workflow, args=''):
args = FixWFArgs(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"],
workflow, args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow + ' ' + args
try:
if not os.path.isdir(basedir):
os.makedirs(basedir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow " + str(
workflow) + " : can't create thread folder: " + str(e)
try:
ret = doCmd(matrixCmd, False, basedir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow " + str(
workflow) + " : caught exception: " + str(e)
return
if __name__ == "__main__":
parser = OptionParser(
usage="%prog -i|--id <jobid> -l|--list <list of workflows>")
parser.add_option("-i",
"--id",
dest="jobid",
help="Job Id e.g. 1of3",
default="1of1")
parser.add_option("-l",
from logreaderUtils import transform_and_write_config_file, add_exception_to_config
def runStep1Only(basedir, workflow, args=''):
args = FixWFArgs(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"],
workflow, args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow + ' ' + args
try:
if not os.path.isdir(workdir):
os.makedirs(workdir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow " + str(
workflow) + " : can't create thread folder: " + str(e)
try:
ret = doCmd(matrixCmd, False, workdir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation steps, workflow " + str(
workflow) + " : caught exception: " + str(e)
return
def runThreadMatrix(basedir,
workflow,
args='',
logger=None,
force=False,
wf_err={}):
if (not force) and logger and logger.relvalAlreadyDone(workflow):
print "Message>> Not runing workflow ", workflow, " as it is already ran"
return
#! /usr/bin/env python
from sys import exit, argv
from os import environ
from cmsutils import cmsRunProcessCount, doCmd
from logUpdater import LogUpdater
if (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
exit(1)
timeout=7200
try: timeout=int(argv[1])
except: timeout=7200
logger = LogUpdater(environ["CMSSW_BASE"])
ret = doCmd('cd %s; rm -rf addOnTests; timeout %s addOnTests.py -j %s 2>&1 >addOnTests.log' % (environ["CMSSW_BASE"], timeout,cmsRunProcessCount))
doCmd('cd '+environ["CMSSW_BASE"]+'/addOnTests/logs; zip -r addOnTests.zip *.log')
logger.updateAddOnTestsLogs()
def isNewRunTheMatrix(self):
e, o = doCmd("runTheMatrix.py --help | grep 'maxSteps=MAXSTEPS' | wc -l")
if e: return False
return o=="1"
json_data={"avg": []}
for tm_str in sorted(json_data["avg"],key=int, reverse=True):
tm=int(tm_str)
if tm > max_tm : max_tm=tm
for wf in json_data["avg"][tm_str]:
if wf in workflows: owf.append([wf,tm])
uwf = []
owfs = [ x[0] for x in owf ]
for wf in workflows:
if not wf in owfs: uwf.append([wf,max_tm])
workflows = uwf + owf
if workflows:
workflows = splitWorkflows(workflows, max_wf)
print(workflows)
on_grid = 0
#if '_DEVEL_X' in environ['CMSSW_VERSION']:
# on_grid = 2
total = len(workflows)
try:
for i in range(1, total+1):
wf=",".join(workflows[i-1])
jobid = str(i)+"of"+str(total)
jobfile = workdir+"/ib-run-relval-"+jobid
doCmd("echo WORKFLOWS="+wf+" >"+jobfile)
doCmd("echo JOBID="+jobid+" >>"+jobfile)
if on_grid>0:
doCmd("echo 'SLAVE_LABELS=(condor&&cpu-8)' >>"+jobfile)
on_grid=on_grid-1
except Exception as e:
print("Error " , e)
#! /usr/bin/env python
from sys import exit
from os import environ
from cmsutils import cmsRunProcessCount, doCmd
from logUpdater import LogUpdater
if (not environ.has_key("CMSSW_BASE")) or (not environ.has_key("SCRAM_ARCH")):
print "ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
exit(1)
logger = LogUpdater(environ["CMSSW_BASE"])
ret = doCmd('cd '+environ["CMSSW_BASE"]+'; rm -rf addOnTests; timeout 5400 addOnTests.py -j '+str(cmsRunProcessCount)+' 2>&1 >addOnTests.log ')
doCmd('cd '+environ["CMSSW_BASE"]+'/addOnTests/logs; zip -r addOnTests.zip *.log')
logger.updateAddOnTestsLogs()
max_tm = 0
with open(RelValtimes) as json_file:
try:
json_data = json.load(json_file)
except:
print "Error reading RelVal Times"
json_data = {"avg": []}
for tm_str in sorted(json_data["avg"], key=int, reverse=True):
tm = int(tm_str)
if tm > max_tm: max_tm = tm
for wf in json_data["avg"][tm_str]:
if wf in workflows: owf.append([wf, tm])
uwf = []
owfs = [x[0] for x in owf]
for wf in workflows:
if not wf in owfs: uwf.append([wf, max_tm])
workflows = uwf + owf
if workflows:
workflows = splitWorkflows(workflows, max_wf)
print workflows
total = len(workflows)
try:
for i in range(1, total + 1):
wf = ",".join(workflows[i - 1])
jobid = str(i) + "of" + str(total)
jobfile = workdir + "/ib-run-relval-" + jobid
doCmd("echo WORKFLOWS=" + wf + " >" + jobfile)
doCmd("echo JOBID=" + jobid + " >>" + jobfile)
except Exception as e:
print "Error ", e
""":"
python_cmd="python3"
python -V >/dev/null 2>&1 && python_cmd="python"
exec ${python_cmd} $0 ${1+"$@"}
"""
from __future__ import print_function
from sys import exit, argv
from os import environ
from cmsutils import cmsRunProcessCount, doCmd
from logUpdater import LogUpdater
if ("CMSSW_BASE" not in environ) or ("SCRAM_ARCH" not in environ):
print(
"ERROR: Unable to file the release environment, please make sure you have set the cmssw environment before calling this script"
)
exit(1)
timeout = 7200
try:
timeout = int(argv[1])
except:
timeout = 7200
logger = LogUpdater(environ["CMSSW_BASE"])
ret = doCmd(
'cd %s; rm -rf addOnTests; timeout %s addOnTests.py -j %s 2>&1 >addOnTests.log'
% (environ["CMSSW_BASE"], timeout, cmsRunProcessCount))
doCmd('cd ' + environ["CMSSW_BASE"] +
'/addOnTests/logs; zip -r addOnTests.zip *.log')
logger.updateAddOnTestsLogs()
json_data = {"avg": []}
for tm_str in sorted(json_data["avg"], key=int, reverse=True):
tm = int(tm_str)
if tm > max_tm: max_tm = tm
for wf in json_data["avg"][tm_str]:
if wf in workflows: owf.append([wf, tm])
uwf = []
owfs = [x[0] for x in owf]
for wf in workflows:
if not wf in owfs: uwf.append([wf, max_tm])
workflows = uwf + owf
if workflows:
workflows = splitWorkflows(workflows, max_wf)
print workflows
on_grid = 0
if '_DEVEL_X' in environ['CMSSW_VERSION']:
on_grid = 2
total = len(workflows)
try:
for i in range(1, total + 1):
wf = ",".join(workflows[i - 1])
jobid = str(i) + "of" + str(total)
jobfile = workdir + "/ib-run-relval-" + jobid
doCmd("echo WORKFLOWS=" + wf + " >" + jobfile)
doCmd("echo JOBID=" + jobid + " >>" + jobfile)
if on_grid > 0:
doCmd("echo 'SLAVE_LABELS=(condor&&cpu-8)' >>" + jobfile)
on_grid = on_grid - 1
except Exception as e:
print "Error ", e
except:
max_wf=100
relval_args = GetMatrixOptions(environ["CMSSW_VERSION"], environ["SCRAM_ARCH"])
matrix = PyRelValsThread(1,environ["CMSSW_BASE"])
workflows = matrix.getWorkFlows(relval_args)
if exists(RelValtimes):
owf = []
max_tm=0
with open(RelValtimes) as json_file:
json_data = json.load(json_file)
for tm_str in sorted(json_data["avg"],key=int, reverse=True):
tm=int(tm_str)
if tm > max_tm : max_tm=tm
for wf in json_data["avg"][tm_str]:
if wf in workflows: owf.append([wf,tm])
uwf = []
owfs = [ x[0] for x in owf ]
for wf in workflows:
if not wf in owfs: uwf.append([wf,max_tm])
workflows = uwf + owf
if workflows:
workflows = splitWorkflows(workflows, max_wf)
print workflows
total = len(workflows)
for i in range(1, total+1):
wf=",".join(workflows[i-1])
jobid = str(i)+"of"+str(total)
jobfile = workdir+"/ib-run-relval-"+jobid
doCmd("echo WORKFLOWS="+wf+" >"+jobfile)
doCmd("echo JOBID="+jobid+" >>"+jobfile)