workflows = []
wfs = []
for wf in [w for w in sys.argv[1].split(",") if w]:
wfs.append(wf)
while wfs:
queryInfo["workflows"] = " OR ".join(wfs[0:50])
wfs = wfs[50:]
wf_hits = es_query(
index='relvals_stats_*',
query=format(
'release:%(release_cycle)s AND architecture:%(architecture)s AND (%(workflows)s)',
**queryInfo),
start_time=int(time() * 1000) - int(86400 * 1000 * 7),
end_time=int(time() * 1000))
stats = es_workflow_stats(wf_hits)
for wf in stats:
wf_weight = 0
for step in stats[wf]:
stat = stats[wf][step]
wf_weight += stat["cpu"]
workflows.append({"workflow": wf, "weight": wf_weight})
order_workflows = []
for item in sorted(workflows, key=itemgetter("weight"), reverse=True):
order_workflows.append([item["workflow"], item["weight"]])
wfs = []
for x in splitWorkflows(order_workflows, 100):
wfs.append(x)
wf_count = len(wfs)
architecture=arch,
workflows=wf_query[4:]),
start_time=1000 * int(time() - (86400 * 30)),
end_time=1000 * int(time()),
scroll=True)
if (not 'hits' in stats) or (not 'hits' in stats['hits']) or (
not stats['hits']['hits']):
xrelease_cycle = str.lower("_".join(cmssw_ver.split("_", 4)[0:3]) +
"_X")
if xrelease_cycle != release_cycle:
release_cycle = xrelease_cycle
print("Retry: Setting release cycle to ", release_cycle)
continue
break
wf_stats = es_workflow_stats(stats)
#Create Jobs
print("Creating jobs.json file ....")
jobs = {}
jobs["final_job"] = "echo All Done"
jobs["final_per_group"] = {
"command": SCRIPT_DIR + "/workflow_final.py %(jobs_results)s",
"cpu": 10,
"rss": 10 * 1024 * 1024,
"time": 30
}
jobs["env"] = {}
jobs["jobs"] = []
e, o = run_cmd("find . -name workflow.log -type f | sed 's|^./||'")
for cmds_log in o.split("\n"):
query=format('(NOT cpu_max:0) AND exit_code:0 AND release:%(release_cycle)s AND architecture:%(architecture)s AND (%(workflows)s)',
release_cycle=release_cycle+"_*",
architecture=arch,
workflows=wf_query[4:]
),
start_time=1000*int(time()-(86400*30)),
end_time=1000*int(time()))
if (not 'hits' in stats) or (not 'hits' in stats['hits']) or (not stats['hits']['hits']):
xrelease_cycle = str.lower("_".join(cmssw_ver.split("_",4)[0:3])+"_X")
if xrelease_cycle!=release_cycle:
release_cycle=xrelease_cycle
print "Retry: Setting release cycle to ",release_cycle
continue
break
wf_stats = es_workflow_stats(stats)
#Create Jobs
print "Creating jobs.json file ...."
jobs = {}
jobs["final_job"] = "echo All Done"
jobs["final_per_group"] = {"command": SCRIPT_DIR+"/workflow_final.py %(jobs_results)s", "cpu": 10, "rss": 10*1024*1024, "time" : 30}
jobs["env"]={}
jobs["jobs"]=[]
e , o = getstatusoutput ("find . -name workflow.log -type f | sed 's|^./||'")
for cmds_log in o.split("\n"):
cmds = os.path.join(os.path.dirname(cmds_log),"wf_steps.txt")
wf = cmds.split("_")[0]
group ={"name": wf, "commands":[]}
if os.path.exists(cmds):
e, o = getstatusoutput ("cat %s | grep ^step" % cmds)