def wfl(wf,v=False,p=False,ms=False):
wfn = wf.name
wfs = wf.wm_status
pid = None
pids=filter(lambda seg: seg.count('-')==2, wf.name.split('_'))
if len(pids):
pid=pids[0]
text=', '.join([
#wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a>'%(wfn,wfn),
'(%s)'%wfs,
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts</a>'%wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">wkl</a>'%wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'%wfn,
'<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>'%wfn,
'<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'%pid,
'<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank">pv</a>'%wfn
])
if p:
wl = getWorkLoad('cmsweb.cern.ch',wfn)
text+=', (%s)'%(wl['RequestPriority'])
if pid:
if ms:
mcm_s = json.loads(os.popen('curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure'%pid).read())[pid]
text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>'%(pid,mcm_s)
else:
text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>'%(pid)
if v and wfs!='acquired':
text+='<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>'%(wfn.replace('_','/'),wfn.replace('_','/'))
text+='<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank"><img src="http://hcc-briantest.unl.edu/prodview/graphs/%s/daily" style="height:50px"></a>'%(wfn,wfn)
return text
def getWL( wfn ):
cached = filter(lambda d : d['RequestName']==wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad(reqmgr_url,wfn)
return wl
def rejector(url, specific ):
if specific.startswith('/'):
pass
else:
wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject",spec
return
results=[]
wl = getWorkLoad(url, wfo.name)
if wl['RequestStatus'] in ['assignment-approved','new']:
#results.append( reqMgrClient.rejectWorkflow(url, wfo.name))
reqMgrClient.rejectWorkflow(url, wfo.name)
else:
#results.append( reqMgrClient.abortWorkflow(url, wfo.name))
reqMgrClient.abortWorkflow(url, wfo.name)
datasets = wl['OutputDatasets']
for dataset in datasets:
results.append( setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', dataset, 'INVALID', None) )
if all(map(lambda result : result in ['None',None],results)):
wfo.status = 'forget'
session.commit()
print wfo.name,"and",datasets,"are rejected"
else:
print "error in rejecting",wfo.name,results
def getWL(wfn):
cached = filter(lambda d: d['RequestName'] == wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad(reqmgr_url, wfn)
return wl
def getWL(wfn):
cached = filter(lambda d: d["RequestName"] == wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad("cmsweb.cern.ch", wfn)
return wl
def completor(url, specific):
wfs = []
wfs.extend( session.query(Workflow).filter(Workflow.status == 'away').all() )
wfs.extend( session.query(Workflow).filter(Workflow.status.startswith('assistance')).all() )
for wfo in wfs:
if specific and not specific in wfo.name: continue
## get all of the same
wl = getWorkLoad(url, wfo.name)
familly = getWorkflowById( url, wl['PrepID'] ,details=True)
for member in familly:
### if member['RequestName'] == wl['RequestName']: continue
if member['RequestDate'] < wl['RequestDate']: continue
if member['RequestStatus'] in ['None',None]: continue
## then set force complete all members
if member['RequestStatus'] in ['running-opened','running-closed']:
print "setting",member['RequestName'],"force-complete"
reqMgrClient.setWorkflowForceComplete(url, member['RequestName'])
def completor(url, specific):
wfs = []
wfs.extend(session.query(Workflow).filter(Workflow.status == "away").all())
wfs.extend(session.query(Workflow).filter(Workflow.status.startswith("assistance")).all())
for wfo in wfs:
if specific and not specific in wfo.name:
continue
## get all of the same
wl = getWorkLoad(url, wfo.name)
familly = getWorkflowById(url, wl["PrepID"], details=True)
for member in familly:
### if member['RequestName'] == wl['RequestName']: continue
if member["RequestDate"] < wl["RequestDate"]:
continue
if member["RequestStatus"] in ["None", None]:
continue
## then set force complete all members
if member["RequestStatus"] in ["running-opened", "running-closed"]:
print "setting", member["RequestName"], "force-complete"
reqMgrClient.setWorkflowForceComplete(url, member["RequestName"])
def wfl(wf,view=False,p=False,ms=False,within=False,ongoing=False,status=False,update=False):
wfn = wf.name
wfs = wf.wm_status
wl = None
pid = None
wl_pid = None
pids=filter(lambda seg: seg.count('-')==2, wf.name.split('_'))
if len(pids):
pids = pids[:1]
pid=pids[0]
if not pids:
wl = getWL( wf.name )
pids = getPrepIDs( wl )
pid = pids[0]
wl_pid = pid
if 'task' in wf.name:
wl_pid = 'task_'+pid
text=', '.join([
#wfn,
#'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a> '%(wfn,wfn),
#'<table><tr><td>%s</td></tr></table>'%(wfn),
#'<span>%s</span>'%(wfn),
"%s "%wfn,
'(%s) <br>'%wfs])
text+=', '.join([
'<a href="https://%s/reqmgr2/fetch?rid=%s" target="_blank">dts</a>'%(reqmgr_url,wfn),
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts-req1</a>'%wfn,
#TOFIX '<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>'%wfn,
'<a href="https://%s/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>'%(reqmgr_url,wfn),
'<a href="https://%s/reqmgr2/data/request?name=%s" target="_blank">req</a>'%(reqmgr_url,wfn),
#'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>'%wfn,
#TOFIX '<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'%wfn,
'<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>'%wfn,
'<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>'%wfn,
'<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'%pid,
'<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank">pv</a>'%wfn,
#deprecated '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>'%wfn,
'<a href="closeout.html#%s" target="_blank">clo</a>'%wfn,
'<a href="statuses.html#%s" target="_blank">st</a>'%wfn,
'<a href="https://%s/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>'%(reqmgr_url,wfn)
])
if within and (not view or wfs=='completed'):
wl = getWL( wfn )
dataset =None
if 'InputDataset' in wl:
dataset = wl['InputDataset']
if 'Task1' in wl and 'InputDataset' in wl['Task1']:
dataset = wl['Task1']['InputDataset']
if dataset:
text+=', '.join(['',
'<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>'%dataset,
'<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>'%dataset,
'<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>'%dataset,
'<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>'%dataset,
])
if p:
cached = filter(lambda d : d['RequestName']==wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad('cmsweb.cern.ch',wfn)
text+=', (%s)'%(wl['RequestPriority'])
pass
if pid:
if ms:
mcm_s = json.loads(os.popen('curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure'%pid).read())[pid]
text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>'%(pid,mcm_s)
else:
text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>'%(pid)
text+=', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>'%(wl_pid)
if status:
if wf.status.startswith('assistance'):
text+=', <a href="assistance.html#%s" target="_blank">assist</a>'%wfn
text+=' : %s '%(wf.status)
if view and wfs!='acquired':
text+='<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>'%(wfn.replace('_','/'),wfn.replace('_','/'))
if ongoing:
text+='<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank"><img src="http://cms-gwmsmon.cern.ch/prodview/graphs/%s/daily" style="height:50px"></a>'%(wfn,wfn)
if ongoing:
date1 = time.strftime('%Y-%m-%d+%H:%M', time.gmtime(time.mktime(time.gmtime())-(15*24*60*60)) )
date2 = time.strftime('%Y-%m-%d+%H:%M', time.gmtime())
text+='<a href="http://dashb-cms-job.cern.ch/dashboard/templates/web-job2/#table=Jobs&date1=%s&date2=%s&sortby=site&task=wmagent_%s">dashb</a>'%( date1, date2, wfn )
if ongoing and wfn in boost:
for task in boost[wfn]:
overflow = boost[wfn][task].get('ReplaceSiteWhitelist',None)
if not overflow:
overflow = boost[wfn][task].get('AddWhitelist',None)
if overflow:
text+=',boost (<a href=equalizor.json>%d</a>)'%len(overflow)
#text+="<hr>"
return text
def injector(url, options, specific):
## passing a round of invalidation of what needs to be invalidated
if options.invalidate:
invalidator(url)
workflows = getWorkflows(url, status=options.wmstatus, user=options.user)
existing = [wf.name for wf in session.query(Workflow).all()]
## browse for assignment-approved requests, browsed for ours, insert the diff
for wf in workflows:
if wf not in existing:
print "putting", wf
new_wf = Workflow(name=wf,
status=options.setstatus,
wm_status=options.wmstatus)
session.add(new_wf)
session.commit()
existing = [wf.name for wf in session.query(Workflow).all()]
## pick up replacements
for wf in session.query(Workflow).filter(
Workflow.status == 'trouble').all():
if specific and wf.name != specific:
continue
print wf.name
wl = getWorkLoad(url, wf.name)
familly = getWorkflowById(url, wl['PrepID'])
if len(familly) == 1:
print wf.name, "ERROR has no replacement"
continue
print wf.name, "has", len(familly), "familly members"
for member in familly:
if member != wf.name:
fwl = getWorkLoad(url, member)
if options.replace:
if member != options.replace: continue
else:
if fwl['RequestDate'] < wl['RequestDate']: continue
if fwl['RequestType'] == 'Resubmission': continue
if fwl['RequestStatus'] in ['None', None]: continue
new_wf = session.query(Workflow).filter(
Workflow.name == member).first()
if not new_wf:
print "putting", member
status = 'away'
if fwl['RequestStatus'] in ['assignment-approved']:
status = 'considered'
new_wf = Workflow(name=member,
status=status,
wm_status=fwl['RequestStatus'])
wf.status = 'forget'
session.add(new_wf)
session.commit()
else:
if new_wf.status == 'forget': continue
print "getting", new_wf.name, "as replacement of", wf.name
for tr in session.query(Transfer).all():
if wf.id in tr.workflows_id:
sw = copy.deepcopy(tr.workflows_id)
sw.remove(wf.id)
sw.append(new_wf.id)
tr.workflows_id = sw
print tr.phedexid, "got", new_wf.name
if new_wf.status != 'away':
new_wf.status = 'staging'
session.commit()
## don't do that automatically
#wf.status = 'forget'
session.commit()
wfs = []
for status in statuses: wfs.extend( session.query(Workflow).filter(Workflow.status==status).all() )
if max_wf: wfs = wfs[:max_wf]
random.shuffle( wfs )
all_blocks_at_sites = defaultdict(set)
done = json.loads(open('myblock_done.json').read())
print len(wfs),"to look the output of"
for iw,wfo in enumerate(wfs):
print "%s/%s:"%(iw,len(wfs)),wfo.name
#wfi = workflowInfo(url, wfo.name)
#outs= wfi.request['OutputDatasets']
wl = getWorkLoad(url, wfo.name)
outs= wl['OutputDatasets']
for out in outs:
blocks_at_sites = getDatasetBlockAndSite(url, out, group="")
deletions = getDatasetOnGoingDeletion(url, out)
if len(deletions):
print "\t\tshould not subscribe with on-going deletions",out
continue
for site,blocks in blocks_at_sites.items():
if 'Buffer' in site or 'Export' in site or 'MSS' in site: continue
all_blocks_at_sites[site].update( blocks )
print "\t",out
print "\t\t",len(blocks_at_sites),"sites",sorted(blocks_at_sites.keys()),"with unsubscribed blocks"
if len(all_blocks_at_sites.keys())==0 and len(wfs):
## no subscription to be done at this time, let me know
def injector(url, options, specific):
use_mcm = True
up = componentInfo( mcm = use_mcm, soft=['mcm'] )
if not up.check(): return
use_mcm = up.status['mcm']
workflows = getWorkflows(url, status=options.wmstatus, user=options.user)
workflows.extend( getWorkflows(url, status=options.wmstatus, user='******', rtype="ReReco")) ## regardless of users, pick up all ReReco on the table
print len(workflows),"in line"
cannot_inject = set()
status_cache = defaultdict(str)
## browse for assignment-approved requests, browsed for ours, insert the diff
for wf in workflows:
if specific and not specific in wf: continue
exists = session.query(Workflow).filter(Workflow.name == wf ).first()
if not exists:
wfi = workflowInfo(url, wf)
#wl = getWorkLoad(url, wf)
## check first that there isn't related here with something valid
can_add = True
## first try at finding a match
# print wfi.request
familly = session.query(Workflow).filter(Workflow.name.contains(wfi.request['PrepID'])).all()
if not familly:
#req_familly = getWorkflowById( url, wl['PrepID'])
#familly = [session.query(Workflow).filter(Workflow.name == member).first() for member in req_familly]
pids = wfi.getPrepIDs()
req_familly = []
for pid in pids:
req_familly.extend( getWorkflowById( url, pid, details=True) )
familly = []
print len(req_familly),"members"
for req_member in req_familly:
#print "member",req_member['RequestName']
owfi = workflowInfo(url, req_member['RequestName'], request=req_member)
other_pids = owfi.getPrepIDs()
if set(pids) == set(other_pids):
## this is a real match
familly.extend( session.query(Workflow).filter(Workflow.name == req_member['RequestName']).all() )
for lwfo in familly:
if lwfo:
## we have it already
if not lwfo.status in ['forget','trouble','forget-unlock','forget-out-unlock']:
sendLog('injector',"Should not put %s because of %s %s"%( wf, lwfo.name,lwfo.status ))
print "Should not put",wf,"because of",lwfo.name,lwfo.status
cannot_inject.add( wf )
can_add = False
## add a check on validity of input datasets
_,prim,par,sec = wfi.getIO()
for d in list(prim)+list(par)+list(sec):
if not d in status_cache:
status_cache[d] = getDatasetStatus(d)
if status_cache[d] != 'VALID':
wfi.sendLog('injector',"One of the input is not VALID. %s : %s"%( d, status_cache[d]))
sendLog('injector',"One of the input of %s is not VALID. %s : %s"%( wf, d, status_cache[d]))
can_add = False
if not can_add: continue
wfi.sendLog('injector',"considering %s"%wf)
new_wf = Workflow( name = wf , status = options.setstatus, wm_status = options.wmstatus)
session.add( new_wf )
session.commit()
time.sleep(0.5)
else:
#print "already have",wf
pass
if cannot_inject:
#sendEmail('workflow duplicates','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject)))
sendLog('injector','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject)), level='warning')
## passing a round of invalidation of what needs to be invalidated
if use_mcm and (options.invalidate or True):
invalidator(url)
no_replacement = set()
## pick up replacements
for wf in session.query(Workflow).filter(Workflow.status == 'trouble').all():
print wf.name
if specific and not specific in wf.name: continue
print wf.name
wfi = workflowInfo(url, wf.name )
wl = wfi.request #getWorkLoad(url, wf.name)
familly = getWorkflowById( url, wl['PrepID'] )
true_familly = []
for member in familly:
if member == wf.name: continue
fwl = getWorkLoad(url , member)
if options.replace:
if member != options.replace: continue
else:
if fwl['RequestDate'] < wl['RequestDate']: continue
if fwl['RequestType']=='Resubmission': continue
if fwl['RequestStatus'] in ['None',None,'new']: continue
if fwl['RequestStatus'] in ['rejected','rejected-archived','aborted','aborted-archived']: continue
true_familly.append( fwl )
if len(true_familly)==0:
#sendLog('injector','%s had no replacement'%wf.name, level='critical')
wfi.sendLog('injector','the workflow was found in trouble with no replacement')
no_replacement.add( wf.name )
continue
else:
wfi.sendLog('injector','the workflow was found in trouble and has a replacement')
print wf.name,"has",len(familly),"familly members"
print wf.name,"has",len(true_familly),"true familly members"
##we cannot have more than one of them !!! pick the last one
if len(true_familly)>1:
#sendEmail('multiple wf','please take a look at injector for %s'%wf.name)
sendLog('injector','Multiple wf in line, will take the last one for %s \n%s'%( wf.name, ', '.join(fwl['RequestName'] for fwl in true_familly)), level='critical')
for fwl in true_familly[-1:]:
member = fwl['RequestName']
new_wf = session.query(Workflow).filter(Workflow.name == member).first()
if not new_wf:
sendLog('injector',"putting %s as replacement of %s"%( member, wf.name))
status = 'away'
if fwl['RequestStatus'] in ['assignment-approved']:
status = 'considered'
new_wf = Workflow( name = member, status = status, wm_status = fwl['RequestStatus'])
wf.status = 'forget'
session.add( new_wf )
else:
if new_wf.status == 'forget': continue
sendLog('injector',"getting %s as replacement of %s"%( new_wf.name, wf.name ))
wf.status = 'forget'
for tr in session.query(Transfer).all():
if wf.id in tr.workflows_id:
sw = copy.deepcopy(tr.workflows_id)
sw.remove( wf.id)
sw.append(new_wf.id)
tr.workflows_id = sw
print tr.phedexid,"got",new_wf.name
if new_wf.status != 'away':
print "\t setting it considered"
new_wf.status = 'considered'
if tr.phedexid<0: ## set it back to positive
tr.phedexid = -tr.phedexid
session.commit()
## don't do that automatically
#wf.status = 'forget'
session.commit()
if no_replacement:
#sendEmail('workflow with no replacement','%s \n are dangling there'%( '\n'.join(no_replacement)))
sendLog('injector','workflow with no replacement, %s \n are dangling there'% ( '\n'.join(no_replacement)), level='critical')
def injector(url, options, specific):
use_mcm = True
up = componentInfo(mcm=use_mcm, soft=["mcm"])
if not up.check():
return
use_mcm = up.status["mcm"]
workflows = getWorkflows(url, status=options.wmstatus, user=options.user)
existing = [wf.name for wf in session.query(Workflow).all()]
## browse for assignment-approved requests, browsed for ours, insert the diff
for wf in workflows:
if wf not in existing:
print "putting", wf
new_wf = Workflow(name=wf, status=options.setstatus, wm_status=options.wmstatus)
session.add(new_wf)
session.commit()
time.sleep(1)
existing = [wf.name for wf in session.query(Workflow).all()]
## passing a round of invalidation of what needs to be invalidated
if use_mcm and (options.invalidate or True):
invalidator(url)
## pick up replacements
for wf in session.query(Workflow).filter(Workflow.status == "trouble").all():
if specific and wf.name != specific:
continue
print wf.name
wl = getWorkLoad(url, wf.name)
familly = getWorkflowById(url, wl["PrepID"])
true_familly = []
for member in familly:
if member == wf.name:
continue
fwl = getWorkLoad(url, member)
if options.replace:
if member != options.replace:
continue
else:
if fwl["RequestDate"] < wl["RequestDate"]:
continue
if fwl["RequestType"] == "Resubmission":
continue
if fwl["RequestStatus"] in ["None", None]:
continue
true_familly.append(fwl)
if len(true_familly) == 0:
print wf.name, "ERROR has no replacement"
known = []
try:
known = json.loads(open("no_replacement.json").read())
except:
pass
if not wf.name in known:
sendEmail(
"workflow in %s with no replacement" % (wl["RequestStatus"]), "%s is dangling there" % (wf.name)
)
known.append(wf.name)
open("no_replacement.json", "w").write(json.dumps(known, indent=2))
continue
print wf.name, "has", len(familly), "familly members"
print wf.name, "has", len(true_familly), "true familly members"
for fwl in true_familly:
member = fwl["RequestName"]
new_wf = session.query(Workflow).filter(Workflow.name == member).first()
if not new_wf:
print "putting", member, "as replacement of", wf.name
status = "away"
if fwl["RequestStatus"] in ["assignment-approved"]:
status = "considered"
new_wf = Workflow(name=member, status=status, wm_status=fwl["RequestStatus"])
wf.status = "forget"
session.add(new_wf)
else:
if new_wf.status == "forget":
continue
print "getting", new_wf.name, "as replacement of", wf.name
wf.status = "forget"
for tr in session.query(Transfer).all():
if wf.id in tr.workflows_id:
sw = copy.deepcopy(tr.workflows_id)
sw.remove(wf.id)
sw.append(new_wf.id)
tr.workflows_id = sw
print tr.phedexid, "got", new_wf.name
if new_wf.status != "away":
print "\t setting it considered"
new_wf.status = "considered"
session.commit()
## don't do that automatically
# wf.status = 'forget'
session.commit()
def closor(url, specific=None):
if not componentInfo().check(): return
CI = campaignInfo()
LI = lockInfo()
## manually closed-out workflows should get to close with checkor
for wfo in session.query(Workflow).filter(Workflow.status=='close').all():
if specific and not specific in wfo.name: continue
## what is the expected #lumis
wl = getWorkLoad(url, wfo.name)
wfo.wm_status = wl['RequestStatus']
if wl['RequestStatus'] in ['announced','normal-archived']:
## manually announced ??
wfo.status = 'done'
wfo.wm_status = wl['RequestStatus']
print wfo.name,"is announced already",wfo.wm_status
session.commit()
expected_lumis = 1
if not 'TotalInputLumis' in wl:
print wfo.name,"has not been assigned yet, or the database is corrupted"
else:
expected_lumis = wl['TotalInputLumis']
## what are the outputs
outputs = wl['OutputDatasets']
## check whether the number of lumis is as expected for each
all_OK = []
#print outputs
if len(outputs):
print wfo.name,wl['RequestStatus']
for out in outputs:
event_count,lumi_count = getDatasetEventsAndLumis(dataset=out)
odb = session.query(Output).filter(Output.datasetname==out).first()
if not odb:
print "adding an output object",out
odb = Output( datasetname = out )
odb.workflow = wfo
session.add( odb )
odb.nlumis = lumi_count
odb.nevents = event_count
odb.workfow_id = wfo.id
if odb.expectedlumis < expected_lumis:
odb.expectedlumis = expected_lumis
else:
expected_lumis = odb.expectedlumis
odb.date = time.mktime(time.gmtime())
session.commit()
print "\t%60s %d/%d = %3.2f%%"%(out,lumi_count,expected_lumis,lumi_count/float(expected_lumis)*100.)
#print wfo.fraction_for_closing, lumi_count, expected_lumis
fraction = wfo.fraction_for_closing
fraction = 0.0
all_OK.append((float(lumi_count) > float(expected_lumis*fraction)))
## only that status can let me go into announced
if wl['RequestStatus'] in ['closed-out']:
print wfo.name,"to be announced"
results=[]#'dummy']
if not results:
for (io,out) in enumerate(outputs):
if all_OK[io]:
results.append(setDatasetStatus(out, 'VALID'))
tier = out.split('/')[-1]
to_DDM = (wl['RequestType'] == 'ReDigi' and not ('DQM' in tier))
campaign = None
try:
campaign = out.split('/')[2].split('-')[0]
except:
if 'Campaign' in wl and wl['Campaign']:
campaign = wl['Campaign']
if campaign and campaign in CI.campaigns and 'toDDM' in CI.campaigns[campaign] and tier in CI.campaigns[campaign]['toDDM']:
to_DDM = True
## inject to DDM when necessary
passed_to_DDM=True
if to_DDM:
#print "Sending",out," to DDM"
status = subprocess.call(['python','assignDatasetToSite.py','--nCopies=2','--dataset='+out,'--exec'])
if status!=0:
print "Failed DDM, retrying a second time"
status = subprocess.call(['python','assignDatasetToSite.py','--nCopies=2','--dataset='+out,'--exec'])
if status!=0:
results.append("Failed DDM for %s"% out)
sendEmail("failed DDM injection","could not add "+out+" to DDM pool. check closor logs.")
passed_to_DDM=False
if passed_to_DDM:
## make a lock release
LI.release_everywhere( out, reason = 'global unlock after passing to DDM')
pass
else:
print wfo.name,"no stats for announcing",out
results.append('No Stats')
if all(map(lambda result : result in ['None',None,True],results)):
## only announce if all previous are fine
results.append(reqMgrClient.announceWorkflowCascade(url, wfo.name))
#print results
if all(map(lambda result : result in ['None',None,True],results)):
wfo.status = 'done'
session.commit()
print wfo.name,"is announced"
else:
print "ERROR with ",wfo.name,"to be announced",json.dumps( results )
else:
print wfo.name,"not good for announcing:",wl['RequestStatus']
def closor(url, specific=None):
CI = campaignInfo()
## manually closed-out workflows should get to close with checkor
for wfo in session.query(Workflow).filter(Workflow.status=='close').all():
if specific and not specific in wfo.name: continue
## what is the expected #lumis
wl = getWorkLoad(url, wfo.name)
wfo.wm_status = wl['RequestStatus']
if wl['RequestStatus'] in ['announced','normal-archived']:
## manually announced ??
wfo.status = 'done'
wfo.wm_status = wl['RequestStatus']
print wfo.name,"is done already",wfo.wm_status
session.commit()
if not 'TotalInputLumis' in wl:
print wfo.name,"has not been assigned yet"
continue
expected_lumis = wl['TotalInputLumis']
## what are the outputs
outputs = wl['OutputDatasets']
## check whether the number of lumis is as expected for each
all_OK = []
#print outputs
if len(outputs):
print wfo.name,wl['RequestStatus']
for out in outputs:
event_count,lumi_count = getDatasetEventsAndLumis(dataset=out)
odb = session.query(Output).filter(Output.datasetname==out).first()
if not odb:
print "adding an output object",out
odb = Output( datasetname = out )
odb.workflow = wfo
session.add( odb )
odb.nlumis = lumi_count
odb.nevents = event_count
odb.workfow_id = wfo.id
if odb.expectedlumis < expected_lumis:
odb.expectedlumis = expected_lumis
else:
expected_lumis = odb.expectedlumis
odb.date = time.mktime(time.gmtime())
session.commit()
print "\t%60s %d/%d = %3.2f%%"%(out,lumi_count,expected_lumis,lumi_count/float(expected_lumis)*100.)
#print wfo.fraction_for_closing, lumi_count, expected_lumis
fraction = wfo.fraction_for_closing
fraction = 0.0
all_OK.append((float(lumi_count) > float(expected_lumis*fraction)))
## only that status can let me go into announced
if wl['RequestStatus'] in ['closed-out']:
print wfo.name,"to be announced"
results=[]#'dummy']
if not results:
results.append(reqMgrClient.announceWorkflowCascade(url, wfo.name))
for (io,out) in enumerate(outputs):
if all_OK[io]:
results.append(setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', out, 'VALID' ,''))
tier = out.split('/')[-1]
to_DDM = (wl['RequestType'] == 'ReDigi' and not ('DQM' in tier))
campaign = None
try:
campaign = out.split('/')[2].split('-')[0]
except:
if 'Campaign' in wl and wl['Campaign']:
campaign = wl['Campaign']
if campaign and campaign in CI.campaigns and 'toDDM' in CI.campaigns[campaign] and tier in CI.campaigns[campaign]['toDDM']:
to_DDM = True
## inject to DDM everything from ReDigi
if to_DDM:
print "Sending",out," to DDM"
subprocess.call(['python','assignDatasetToSite.py','--dataset='+out,'--exec'])
else:
print wfo.name,"no stats for announcing",out
results.append(None)
#print results
if all(map(lambda result : result in ['None',None],results)):
wfo.status = 'done'
session.commit()
print wfo.name,"is announced"
else:
print "ERROR with ",wfo.name,"to be announced",json.dumps( results )
else:
print wfo.name,"not good for announcing:",wl['RequestStatus']
def wfl(wf,
view=False,
p=False,
ms=False,
within=False,
ongoing=False,
status=False,
update=False):
wfn = wf.name
wfs = wf.wm_status
wl = None
pid = None
wl_pid = None
pids = filter(lambda seg: seg.count('-') == 2, wf.name.split('_'))
if len(pids):
pids = pids[:1]
pid = pids[0]
if not pids:
wl = getWL(wf.name)
pids = getPrepIDs(wl)
pid = pids[0]
wl_pid = pid
if 'task' in wf.name:
wl_pid = 'task_' + pid
text = ', '.join([
#wfn,
#'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a> '%(wfn,wfn),
#'<table><tr><td>%s</td></tr></table>'%(wfn),
#'<span>%s</span>'%(wfn),
"%s " % wfn,
'(%s) <br>' % wfs
])
text += ', '.join([
'<a href="https://%s/reqmgr2/fetch?rid=%s" target="_blank">dts</a>'
% (reqmgr_url, wfn),
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts-req1</a>'
% wfn,
#TOFIX '<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>'%wfn,
'<a href="https://%s/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>'
% (reqmgr_url, wfn),
'<a href="https://%s/reqmgr2/data/request?name=%s" target="_blank">req</a>'
% (reqmgr_url, wfn),
#'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>'%wfn,
#TOFIX '<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'%wfn,
'<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>'
% wfn,
'<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>'
% wfn,
'<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'
% pid,
'<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank">pv</a>'
% wfn,
#deprecated '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>'%wfn,
'<a href="closeout.html#%s" target="_blank">clo</a>' % wfn,
'<a href="statuses.html#%s" target="_blank">st</a>' % wfn,
'<a href="https://%s/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>'
% (reqmgr_url, wfn)
])
if within and (not view or wfs == 'completed'):
wl = getWL(wfn)
dataset = None
if 'InputDataset' in wl:
dataset = wl['InputDataset']
if 'Task1' in wl and 'InputDataset' in wl['Task1']:
dataset = wl['Task1']['InputDataset']
if dataset:
text += ', '.join([
'',
'<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>'
% dataset,
'<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>'
% dataset,
'<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>'
% dataset,
'<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>'
% dataset,
])
if p:
cached = filter(lambda d: d['RequestName'] == wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad('cmsweb.cern.ch', wfn)
text += ', (%s)' % (wl['RequestPriority'])
pass
if pid:
if ms:
mcm_s = json.loads(
os.popen(
'curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure'
% pid).read())[pid]
text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>' % (
pid, mcm_s)
else:
text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>' % (
pid)
text += ', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>' % (
wl_pid)
if status:
if wf.status.startswith('assistance'):
text += ', <a href="assistance.html#%s" target="_blank">assist</a>' % wfn
text += ' : %s ' % (wf.status)
if view and wfs != 'acquired':
text += '<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>' % (
wfn.replace('_', '/'), wfn.replace('_', '/'))
if ongoing:
text += '<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank"><img src="http://cms-gwmsmon.cern.ch/prodview/graphs/%s/daily" style="height:50px"></a>' % (
wfn, wfn)
if ongoing:
date1 = time.strftime(
'%Y-%m-%d+%H:%M',
time.gmtime(time.mktime(time.gmtime()) - (15 * 24 * 60 * 60)))
date2 = time.strftime('%Y-%m-%d+%H:%M', time.gmtime())
text += '<a href="http://dashb-cms-job.cern.ch/dashboard/templates/web-job2/#table=Jobs&date1=%s&date2=%s&sortby=site&task=wmagent_%s">dashb</a>' % (
date1, date2, wfn)
if ongoing and wfn in boost:
for task in boost[wfn]:
overflow = boost[wfn][task].get('ReplaceSiteWhitelist', None)
if not overflow:
overflow = boost[wfn][task].get('AddWhitelist', None)
if overflow:
text += ',boost (<a href=equalizor.json>%d</a>)' % len(
overflow)
#text+="<hr>"
return text
from assignSession import *
from utils import getWorkLoad
import sys
if __name__ == "__main__":
spec = sys.argv[1]
status = None
if len(sys.argv)>2:
status = sys.argv[2]
for wf in session.query(Workflow).all():
if spec and spec not in wf.name: continue
#if not wf.status in ['away']: continue
wl = getWorkLoad('cmsweb.cern.ch', wf.name)
wf.wm_status = wl['RequestStatus']
if status:
if status == 'DELETE':
print "removing",wf.name
session.delete( wf )
else:
wf.status = status
elif wf.wm_status in ['assignment-approved']:
wf.status = 'considered'
elif wf.wm_status in ['assigned','acquired','running-closed','running-open','completed']:
if not wf.status.startswith('assistance'):
wf.status = 'away'
else:
print wf.name,"is still in",wf.status
from utils import getWorkLoad, reqmgr_url
import sys
url = reqmgr_url
if __name__ == "__main__":
spec = sys.argv[1]
status = None
if len(sys.argv)>2:
status = sys.argv[2]
for wf in session.query(Workflow).all():
if spec and spec not in wf.name: continue
#if not wf.status in ['away']: continue
wl = getWorkLoad(url, wf.name)
wf.wm_status = wl['RequestStatus']
if status:
if status == 'DELETE':
print "removing",wf.name
session.delete( wf )
else:
wf.status = status
elif wf.wm_status in ['assignment-approved']:
wf.status = 'considered'
elif wf.wm_status in ['assigned','acquired','running-closed','running-open','completed']:
if not wf.status.startswith('assistance'):
wf.status = 'away'
else:
print wf.name,"is still in",wf.status
def cleanor(url, specific=None):
print "Deprecated"
return
if duplicateLock() : return
delete_per_site = {}
do_not_autoapprove = []#'T2_FR_CCIN2P3']
SI = siteInfo()
CI = campaignInfo()
LI = lockInfo()
counts=0
for wfo in session.query(Workflow).filter(Workflow.status == 'done').all():
keep_a_copy = False
if specific and not specific in wfo.name: continue
## what was in input
wl = getWorkLoad(url, wfo.name )
if 'Campaign' in wl and wl['Campaign'] in CI.campaigns and 'clean-in' in CI.campaigns[wl['Campaign']] and CI.campaigns[wl['Campaign']]['clean-in']==False:
print "Skipping cleaning on input for campaign",wl['Campaign'], "as per campaign configuration"
continue
dataset= 'N/A'
if 'InputDataset' in wl:
dataset = wl['InputDataset']
print dataset,"in input"
#print json.dumps(wl, indent=2)
announced_log = filter(lambda change : change["Status"] in ["closed-out","normal-archived","announced"],wl['RequestTransition'])
if not announced_log:
print "Cannot figure out when",wfo.name,"was finished"
continue
now = time.mktime(time.gmtime()) / (60*60*24.)
then = announced_log[-1]['UpdateTime'] / (60.*60.*24.)
if (now-then) <2:
print "workflow",wfo.name, "finished",now-then,"days ago. Too fresh to clean"
continue
else:
print "workflow",wfo.name,"has finished",now-then,"days ago."
if not 'InputDataset' in wl:
## should we set status = clean ? or something even further
print "passing along",wfo.name,"with no input"
wfo.status = 'clean'
session.commit()
continue
if 'MinBias' in dataset:
print "Should not clean anything using",dataset,"setting status further"
wfo.status = 'clean'
session.commit()
continue
total_size = getDatasetSize( dataset ) ## in Gb
#if counts> 20: break
counts+=1
## find any location it is at
our_presence = getDatasetPresence(url, dataset, complete=None, group="DataOps")
also_our_presence = getDatasetPresence(url, dataset, complete=None, group="")
## is there a custodial !!!
custodials = findCustodialLocation(url, dataset)
if not len(custodials):
print dataset,"has no custodial site yet, excluding from cleaning"
continue
## find out whether it is still in use
using_the_same = getWorkflowByInput(url, dataset, details=True)
conflict=False
for other in using_the_same:
if other['RequestName'] == wfo.name: continue
if other['RequestType'] == 'Resubmission': continue
if not other['RequestStatus'] in ['announced','normal-archived','aborted','rejected','aborted-archived','aborted-completed','rejected-archived','closed-out','None',None,'new']:
print other['RequestName'],'is in status',other['RequestStatus'],'preventing from cleaning',dataset
conflict=True
break
if 'Campaign' in other and other['Campaign'] in CI.campaigns and 'clean-in' in CI.campaigns[other['Campaign']] and CI.campaigns[other['Campaign']]['clean-in']==False:
print other['RequestName'],'is in campaign',other['Campaign']
conflict = True
break
if conflict: continue
print "other statuses:",[other['RequestStatus'] for other in using_the_same if other['RequestName'] != wfo.name]
## find all disks
to_be_cleaned = filter(lambda site : site.startswith('T2') or site.endswith('Disk') ,our_presence.keys())
to_be_cleaned.extend( filter(lambda site : site.startswith('T2') or site.endswith('Disk') ,also_our_presence.keys()))
print to_be_cleaned,"for",total_size,"GB"
anaops_presence = getDatasetPresence(url, dataset, complete=None, group="AnalysisOps")
own_by_anaops = anaops_presence.keys()
print "Own by analysis ops and vetoing"
print own_by_anaops
## need to black list the sites where there is a copy of analysis ops
to_be_cleaned = [site for site in to_be_cleaned if not site in own_by_anaops ]
## keep one copy out there
if 'Campaign' in wl and wl['Campaign'] in CI.campaigns and 'keep-one' in CI.campaigns[wl['Campaign']] and CI.campaigns[wl['Campaign']]['keep-one']==True:
print "Keeping a copy of input for",wl['Campaign']
keep_a_copy = True
if keep_a_copy:
keep_at = None
full_copies = [site for (site,(there,_)) in our_presence.items() if there and site.startswith('T1')]
full_copies.extend( [site for (site,(there,_)) in also_our_presence.items() if there and site.startswith('T1')] )
if not full_copies:
full_copies = [site for (site,(there,_)) in our_presence.items() if there and site.startswith('T2')]
full_copies.extend( [site for (site,(there,_)) in also_our_presence.items() if there and site.startswith('T2')] )
if full_copies:
keep_at = random.choice( full_copies )
if not keep_at:
print "We are enable to find a place to keep a full copy of",dataset,"skipping"
continue
else:
## keeping that copy !
print "Keeping a full copy of",dataset,"at",keep_at,"not setting the status further"
to_be_cleaned.remove( keep_at )
else:
wfo.status = 'clean'
## collect delete request per site
for site in to_be_cleaned :
if not site in delete_per_site: delete_per_site[site] = []
if not dataset in [existing[0] for existing in delete_per_site[site]]:
delete_per_site[site].append( (dataset, total_size) )
session.commit()
#open('deletes.json','w').write( json.dumps(delete_per_site,indent=2) )
print json.dumps(delete_per_site, indent=2)
print "\n\n ------- \n\n"
## unroll the deletion per site
## maybe find the optimum site/dataset dataset/site to limit the number of ph requests
for site in delete_per_site:
dataset_list = [info[0] for info in delete_per_site[site]]
size_removal = sum([info[1] for info in delete_per_site[site]]) / 1024.
if site in SI.disk:
free = SI.disk[site]
print site,"has",size_removal,"TB of potential cleanup.",free,"TB available."
else:
print site,"has",size_removal,"TB of potential cleanup. no info on available."
print "\t",','.join(dataset_list)
## make deletion requests
for site in delete_per_site:
site_datasets = [info[0] for info in delete_per_site[site]]
is_tape = any([v in site for v in ['MSS','Export','Buffer'] ])
#comments="Cleanup input after production. DataOps will take care of approving it."
#if is_tape:
# comments="Cleanup input after production."
for item in site_datasets:
LI.release( item, site, 'cleanup of input after production')
def wfl(wf, view=False, p=False, ms=False, within=False, ongoing=False, status=False, update=False):
wfn = wf.name
wfs = wf.wm_status
wl = None
pid = None
pids = filter(lambda seg: seg.count("-") == 2, wf.name.split("_"))
if len(pids):
pids = pids[:1]
pid = pids[0]
if not pids:
wl = getWL(wf.name)
pids = getPrepIDs(wl)
pid = pids[0]
text = ", ".join(
[
# wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a>' % (wfn, wfn),
"(%s) <br>" % wfs,
]
)
text += ", ".join(
[
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts</a>' % wfn,
'<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>' % wfn,
'<a href="https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>' % wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>' % wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>' % wfn,
'<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>' % wfn,
'<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>' % wfn,
'<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'
% pid,
'<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank">pv</a>' % wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>'
% wfn,
'<a href="closeout.html#%s" target="_blank">clo</a>' % wfn,
'<a href="statuses.html#%s" target="_blank">st</a>' % wfn,
'<a href="https://cmsweb.cern.ch/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>'
% wfn,
]
)
if within and (not view or wfs == "completed"):
wl = getWL(wfn)
dataset = None
if "InputDataset" in wl:
dataset = wl["InputDataset"]
if "Task1" in wl and "InputDataset" in wl["Task1"]:
dataset = wl["Task1"]["InputDataset"]
if dataset:
text += ", ".join(
[
"",
"<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>" % dataset,
"<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>"
% dataset,
"<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>"
% dataset,
"<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>"
% dataset,
]
)
if p:
cached = filter(lambda d: d["RequestName"] == wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad("cmsweb.cern.ch", wfn)
text += ", (%s)" % (wl["RequestPriority"])
pass
if pid:
if ms:
mcm_s = json.loads(
os.popen(
"curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure" % pid
).read()
)[pid]
text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>' % (
pid,
mcm_s,
)
else:
text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>' % (pid)
text += (
', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>'
% (pid)
)
if status:
if wf.status.startswith("assistance"):
text += ', <a href="assistance.html#%s" target="_blank">assist</a>' % wfn
text += " : %s " % (wf.status)
if view and wfs != "acquired":
text += (
'<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>'
% (wfn.replace("_", "/"), wfn.replace("_", "/"))
)
if ongoing:
text += (
'<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank"><img src="http://cms-gwmsmon.cern.ch/prodview/graphs/%s/daily" style="height:50px"></a>'
% (wfn, wfn)
)
if ongoing:
date1 = time.strftime("%Y-%m-%d+%H:%M", time.gmtime(time.mktime(time.gmtime()) - (15 * 24 * 60 * 60)))
date2 = time.strftime("%Y-%m-%d+%H:%M", time.gmtime())
text += (
'<a href="http://dashb-cms-job.cern.ch/dashboard/templates/web-job2/#table=Jobs&date1=%s&date2=%s&sortby=site&task=wmagent_%s">dashb</a>'
% (date1, date2, wfn)
)
text += "<hr>"
return text
from assignSession import *
from utils import getWorkLoad
import sys
if __name__ == "__main__":
spec = sys.argv[1]
status = None
if len(sys.argv) > 2:
status = sys.argv[2]
for wf in session.query(Workflow).all():
if spec and spec not in wf.name: continue
#if not wf.status in ['away']: continue
wl = getWorkLoad('cmsweb.cern.ch', wf.name)
wf.wm_status = wl['RequestStatus']
if status:
wf.status = status
elif wf.wm_status in ['assignment-approved']:
wf.status = 'considered'
elif wf.wm_status in [
'assigned', 'acquired', 'running-closed', 'running-open',
'completed'
]:
if not wf.status.startswith('assistance'):
wf.status = 'away'
else:
print wf.name, "is still in", wf.status
elif wf.wm_status in ['closed-out']:
wf.status = 'close'
if max_wf: wfs = wfs[:max_wf]
random.shuffle( wfs )
all_blocks_at_sites = defaultdict(set)
#done = json.loads(open('myblock_done.json').read())
done = {}
print len(wfs),"to look the output of"
for iw,wfn in enumerate(wfs):
if type(wfn)==dict:
wl = wfn
wfn = wl['RequestName']
else:
wl = getWorkLoad(url, wfn)
print "%s/%s:"%(iw,len(wfs)),wfn
if not wl:
continue
outs= wl['OutputDatasets']
for out in outs:
blocks_at_sites = getDatasetBlockAndSite(url, out, group="")
deletions = getDatasetOnGoingDeletion(url, out)
if len(deletions):
print "\t\tshould not subscribe with on-going deletions",out
continue
for site,blocks in blocks_at_sites.items():
if 'Buffer' in site or 'Export' in site or 'MSS' in site: continue
all_blocks_at_sites[site].update( blocks )
def wfl(wf,view=False,p=False,ms=False,within=False,ongoing=False,status=False,update=False):
wfn = wf.name
wfs = wf.wm_status
pid = None
pids=filter(lambda seg: seg.count('-')==2, wf.name.split('_'))
if len(pids):
pid=pids[0]
text=', '.join([
#wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a>'%(wfn,wfn),
'(%s) <br>'%wfs])
text+=', '.join([
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts</a>'%wfn,
'<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>'%wfn,
'<a href="https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>'%wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>'%wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'%wfn,
'<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>'%wfn,
'<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>'%wfn,
'<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'%pid,
'<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank">pv</a>'%wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>'%wfn,
'<a href="closeout.html#%s" target="_blank">clo</a>'%wfn,
'<a href="statuses.html#%s" target="_blank">st</a>'%wfn,
'<a href="https://cmsweb.cern.ch/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>'%wfn
])
if within and (not view or wfs=='completed'):
cached = filter(lambda d : d['RequestName']==wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad('cmsweb.cern.ch',wfn)
if 'InputDataset' in wl:
dataset = wl['InputDataset']
text+=', '.join(['',
'<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>'%dataset,
'<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>'%dataset,
'<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>'%dataset,
'<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>'%dataset,
])
if p:
cached = filter(lambda d : d['RequestName']==wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad('cmsweb.cern.ch',wfn)
text+=', (%s)'%(wl['RequestPriority'])
pass
if pid:
if ms:
mcm_s = json.loads(os.popen('curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure'%pid).read())[pid]
text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>'%(pid,mcm_s)
else:
text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>'%(pid)
text+=', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>'%(pid)
if status:
if wf.status.startswith('assistance'):
text+=', <a href="assistance.html#%s" target="_blank">assist</a>'%wfn
text+=' : %s '%(wf.status)
if view and wfs!='acquired':
text+='<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>'%(wfn.replace('_','/'),wfn.replace('_','/'))
if ongoing:
text+='<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank"><img src="http://hcc-briantest.unl.edu/prodview/graphs/%s/daily" style="height:50px"></a>'%(wfn,wfn)
text+="<hr>"
return text
if max_wf: wfs = wfs[:max_wf]
random.shuffle(wfs)
all_blocks_at_sites = defaultdict(set)
#done = json.loads(open('myblock_done.json').read())
done = {}
print len(wfs), "to look the output of"
for iw, wfn in enumerate(wfs):
if type(wfn) == dict:
wl = wfn
wfn = wl['RequestName']
else:
wl = getWorkLoad(url, wfn)
print "%s/%s:" % (iw, len(wfs)), wfn
if not wl:
continue
outs = wl['OutputDatasets']
for out in outs:
blocks_at_sites = getDatasetBlockAndSite(url, out, group="")
deletions = getDatasetOnGoingDeletion(url, out)
if len(deletions):
print "\t\tshould not subscribe with on-going deletions", out
continue
for site, blocks in blocks_at_sites.items():
if 'Buffer' in site or 'Export' in site or 'MSS' in site: continue
all_blocks_at_sites[site].update(blocks)
def injector(url, options, specific):
use_mcm = True
up = componentInfo(mcm=use_mcm, soft=['mcm'])
if not up.check(): return
use_mcm = up.status['mcm']
workflows = getWorkflows(url, status=options.wmstatus, user=options.user)
workflows.extend(
getWorkflows(url,
status=options.wmstatus,
user='******',
rtype="ReReco")
) ## regardless of users, pick up all ReReco on the table
print len(workflows), "in line"
cannot_inject = set()
## browse for assignment-approved requests, browsed for ours, insert the diff
for wf in workflows:
if specific and not specific in wf: continue
exists = session.query(Workflow).filter(Workflow.name == wf).first()
if not exists:
wfi = workflowInfo(url, wf)
#wl = getWorkLoad(url, wf)
## check first that there isn't related here with something valid
can_add = True
## first try at finding a match
# print wfi.request
familly = session.query(Workflow).filter(
Workflow.name.contains(wfi.request['PrepID'])).all()
if not familly:
#req_familly = getWorkflowById( url, wl['PrepID'])
#familly = [session.query(Workflow).filter(Workflow.name == member).first() for member in req_familly]
pids = wfi.getPrepIDs()
req_familly = []
for pid in pids:
req_familly.extend(getWorkflowById(url, pid, details=True))
familly = []
print len(req_familly), "members"
for req_member in req_familly:
#print "member",req_member['RequestName']
owfi = workflowInfo(url,
req_member['RequestName'],
request=req_member)
other_pids = owfi.getPrepIDs()
if set(pids) == set(other_pids):
## this is a real match
familly.extend(
session.query(Workflow).filter(
Workflow.name ==
req_member['RequestName']).all())
for lwfo in familly:
if lwfo:
## we have it already
if not lwfo.status in [
'forget', 'trouble', 'forget-unlock',
'forget-out-unlock'
]:
sendLog(
'injector', "Should not put %s because of %s %s" %
(wf, lwfo.name, lwfo.status))
print "Should not put", wf, "because of", lwfo.name, lwfo.status
cannot_inject.add(wf)
can_add = False
if not can_add: continue
wfi.sendLog('injector', "considering %s" % wf)
new_wf = Workflow(name=wf,
status=options.setstatus,
wm_status=options.wmstatus)
session.add(new_wf)
session.commit()
time.sleep(0.5)
else:
#print "already have",wf
pass
if cannot_inject:
#sendEmail('workflow duplicates','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject)))
sendLog(
'injector',
'These workflow cannot be added in because of duplicates \n\n %s' %
('\n'.join(cannot_inject)),
level='warning')
## passing a round of invalidation of what needs to be invalidated
if use_mcm and (options.invalidate or True):
invalidator(url)
no_replacement = set()
## pick up replacements
for wf in session.query(Workflow).filter(
Workflow.status == 'trouble').all():
print wf.name
if specific and not specific in wf.name: continue
print wf.name
wfi = workflowInfo(url, wf.name)
wl = wfi.request #getWorkLoad(url, wf.name)
familly = getWorkflowById(url, wl['PrepID'])
true_familly = []
for member in familly:
if member == wf.name: continue
fwl = getWorkLoad(url, member)
if options.replace:
if member != options.replace: continue
else:
if fwl['RequestDate'] < wl['RequestDate']: continue
if fwl['RequestType'] == 'Resubmission': continue
if fwl['RequestStatus'] in ['None', None, 'new']: continue
if fwl['RequestStatus'] in [
'rejected', 'rejected-archived', 'aborted',
'aborted-archived'
]:
continue
true_familly.append(fwl)
if len(true_familly) == 0:
#sendLog('injector','%s had no replacement'%wf.name, level='critical')
wfi.sendLog(
'injector',
'the workflow was found in trouble with no replacement')
no_replacement.add(wf.name)
continue
else:
wfi.sendLog(
'injector',
'the workflow was found in trouble and has a replacement')
print wf.name, "has", len(familly), "familly members"
print wf.name, "has", len(true_familly), "true familly members"
##we cannot have more than one of them !!! pick the last one
if len(true_familly) > 1:
#sendEmail('multiple wf','please take a look at injector for %s'%wf.name)
sendLog('injector',
'Multiple wf in line, will take the last one for %s \n%s' %
(wf.name, ', '.join(fwl['RequestName']
for fwl in true_familly)),
level='critical')
for fwl in true_familly[-1:]:
member = fwl['RequestName']
new_wf = session.query(Workflow).filter(
Workflow.name == member).first()
if not new_wf:
sendLog('injector',
"putting %s as replacement of %s" % (member, wf.name))
status = 'away'
if fwl['RequestStatus'] in ['assignment-approved']:
status = 'considered'
new_wf = Workflow(name=member,
status=status,
wm_status=fwl['RequestStatus'])
wf.status = 'forget'
session.add(new_wf)
else:
if new_wf.status == 'forget': continue
sendLog(
'injector',
"getting %s as replacement of %s" % (new_wf.name, wf.name))
wf.status = 'forget'
for tr in session.query(Transfer).all():
if wf.id in tr.workflows_id:
sw = copy.deepcopy(tr.workflows_id)
sw.remove(wf.id)
sw.append(new_wf.id)
tr.workflows_id = sw
print tr.phedexid, "got", new_wf.name
if new_wf.status != 'away':
print "\t setting it considered"
new_wf.status = 'considered'
if tr.phedexid < 0: ## set it back to positive
tr.phedexid = -tr.phedexid
session.commit()
## don't do that automatically
#wf.status = 'forget'
session.commit()
if no_replacement:
#sendEmail('workflow with no replacement','%s \n are dangling there'%( '\n'.join(no_replacement)))
sendLog('injector',
'workflow with no replacement, %s \n are dangling there' %
('\n'.join(no_replacement)),
level='critical')
def injector(url, options, specific):
mlock = moduleLock()
if mlock(): return
use_mcm = True
up = componentInfo(soft=['mcm','wtc','jira'] )
if not up.check(): return
use_mcm = up.status['mcm']
UC = unifiedConfiguration()
transform_keywords = UC.get('convert_to_stepchain')
workflows = getWorkflows(url, status=options.wmstatus, user=options.user)
for user in UC.get("user_rereco"):
workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="ReReco"))
for user in (options.user_relval.split(',') if options.user_relval else UC.get("user_relval")) :
workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="TaskChain"))
for user in (options.user_storeresults.split(',') if options.user_storeresults else UC.get("user_storeresults")) :
workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="StoreResults"))
print len(workflows),"in line"
cannot_inject = set()
to_convert = set()
status_cache = defaultdict(str)
## browse for assignment-approved requests, browsed for ours, insert the diff
for wf in workflows:
if specific and not specific in wf: continue
exists = session.query(Workflow).filter(Workflow.name == wf ).first()
if not exists:
wfi = workflowInfo(url, wf)
## check first that there isn't related here with something valid
can_add = True
## first try at finding a match
familly = session.query(Workflow).filter(Workflow.name.contains(wfi.request['PrepID'])).all()
if not familly:
pids = wfi.getPrepIDs()
req_familly = []
for pid in pids:
req_familly.extend( getWorkflowById( url, pid, details=True) )
familly = []
print len(req_familly),"members"
for req_member in req_familly:
#print "member",req_member['RequestName']
owfi = workflowInfo(url, req_member['RequestName'], request=req_member)
other_pids = owfi.getPrepIDs()
if set(pids) == set(other_pids):
## this is a real match
familly.extend( session.query(Workflow).filter(Workflow.name == req_member['RequestName']).all() )
for lwfo in familly:
if lwfo:
## we have it already
if not lwfo.status in ['forget','trouble','forget-unlock','forget-out-unlock']:
wfi.sendLog('injector',"Should not put %s because of %s %s"%( wf, lwfo.name,lwfo.status ))
sendLog('injector',"Should not put %s because of %s %s"%( wf, lwfo.name,lwfo.status ), level='critical')
print "Should not put",wf,"because of",lwfo.name,lwfo.status
cannot_inject.add( wf )
can_add = False
## add a check on validity of input datasets
_,prim,par,sec = wfi.getIO()
for d in list(prim)+list(par)+list(sec):
if not d in status_cache:
status_cache[d] = getDatasetStatus(d)
if status_cache[d] != 'VALID':
wfi.sendLog('injector',"One of the input is not VALID. %s : %s"%( d, status_cache[d]))
sendLog('injector',"One of the input of %s is not VALID. %s : %s"%( wf, d, status_cache[d]), level='critical')
can_add = False
#else:
# ##make sure that all blocks get closed
# closeAllBlocks(url, d)
## check for any file in phedex, to verify existence
_,ph_files,_,_ = getDatasetFiles(url, d)
if not ph_files and not ( 'StoreResults' == wfi.request.setdefault('RequestType',None) ):
wfi.sendLog('injector',"One of the input has no file in phedex: %s" % d )
sendLog('injector',"One of the input has no file in phedex: %s"% d, level='critical')
can_add = False
### ban some workflow that you don't like anymore
#outputs = wfi.request['OutputDatasets']
if not can_add: continue
## temporary hack to transform specific taskchain into stepchains
good_for_stepchain = wfi.isGoodToConvertToStepChain( keywords = transform_keywords)
#good_for_stepchain = wfi.isGoodToConvertToStepChain( keywords = None)
## match keywords and technical constraints
if (not options.no_convert) and good_for_stepchain and not wfi.isRelval():
to_convert.add( wf )
wfi.sendLog('injector','Transforming %s TaskChain into StepChain'%wf)
sendEmail('convertion to stepchain','Transforming %s TaskChain into StepChain'%wf)
wfi.sendLog('injector',"considering %s"%wf)
new_wf = Workflow( name = wf , status = options.setstatus, wm_status = options.wmstatus)
session.add( new_wf )
session.commit()
time.sleep(0.5)
else:
#print "already have",wf
pass
if cannot_inject:
#sendEmail('workflow duplicates','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject)))
sendLog('injector','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject)), level='critical')
for wf in to_convert:
os.system('./Unified/rejector.py --clone --to_step --comments \"Transform to StepChain\" %s'% wf)
## passing a round of invalidation of what needs to be invalidated
if use_mcm and (options.invalidate or True):
invalidator(url)
no_replacement = set()
#print "getting all transfers"
#all_transfers=session.query(Transfer).all()
#print "go!"
## pick up replacements
for wf in session.query(Workflow).filter(Workflow.status == 'trouble').all():
print wf.name
if specific and not specific in wf.name: continue
print wf.name
wfi = workflowInfo(url, wf.name )
wl = wfi.request #getWorkLoad(url, wf.name)
familly = getWorkflowById( url, wl['PrepID'] )
true_familly = []
for member in familly:
if member == wf.name: continue
fwl = getWorkLoad(url , member)
if options.replace:
if member != options.replace: continue
else:
if fwl['RequestDate'] < wl['RequestDate']: continue
if fwl['RequestType']=='Resubmission': continue
if fwl['RequestStatus'] in ['None',None,'new']: continue
if fwl['RequestStatus'] in ['rejected','rejected-archived','aborted','aborted-archived']: continue
true_familly.append( fwl )
if len(true_familly)==0:
#sendLog('injector','%s had no replacement'%wf.name, level='critical')
if wfi.isRelval():
#wfi.sendLog('injector','the workflow was found in trouble with no replacement. As a relval, there is no clean way to handle this.')
wfi.sendLog('injector','the workflow was found in trouble with no replacement. As a relval, there is no clean way to handle this. Setting forget')
wf.status = 'forget'
session.commit()
else:
wfi.sendLog('injector','the workflow was found in trouble with no replacement')
no_replacement.add( wf.name )
continue
else:
wfi.sendLog('injector','the workflow was found in trouble and has a replacement')
print wf.name,"has",len(familly),"familly members"
print wf.name,"has",len(true_familly),"true familly members"
##we cannot have more than one of them !!! pick the last one
if len(true_familly)>1:
#sendEmail('multiple wf','please take a look at injector for %s'%wf.name)
sendLog('injector','Multiple wf in line, will take the last one for %s \n%s'%( wf.name, ', '.join(fwl['RequestName'] for fwl in true_familly)), level='critical')
for fwl in true_familly[-1:]:
member = fwl['RequestName']
new_wf = session.query(Workflow).filter(Workflow.name == member).first()
if not new_wf:
sendLog('injector',"putting %s as replacement of %s"%( member, wf.name))
status = 'away'
if fwl['RequestStatus'] in ['assignment-approved']:
status = 'considered'
new_wf = Workflow( name = member, status = status, wm_status = fwl['RequestStatus'])
wf.status = 'forget'
session.add( new_wf )
else:
if new_wf.status == 'forget': continue
sendLog('injector',"getting %s as replacement of %s"%( new_wf.name, wf.name ))
wf.status = 'forget'
for tr in session.query(TransferImp).filter( TransferImp.workflow_id == wf.id).all():
## get all transfer working for the old workflow
existing = session.query(TransferImp).filter( TransferImp.phedexid == tr.phedexid).filter( TransferImp.workflow_id == new_wf.id).all()
tr.active = False ## disable the old one
if not existing:
## create the transfer object for the new dependency
tri = TransferImp( phedexid = tr.phedexid,
workflow = new_wf)
session.add( tri )
session.commit()
## don't do that automatically
#wf.status = 'forget'
session.commit()
if no_replacement:
#sendEmail('workflow with no replacement','%s \n are dangling there'%( '\n'.join(no_replacement)))
sendLog('injector','workflow with no replacement\n%s \n are dangling there'% ( '\n'.join(no_replacement)), level='critical')
def injector(url, options, specific):
## passing a round of invalidation of what needs to be invalidated
if options.invalidate:
invalidator(url)
workflows = getWorkflows(url, status=options.wmstatus,user=options.user)
existing = [wf.name for wf in session.query(Workflow).all()]
## browse for assignment-approved requests, browsed for ours, insert the diff
for wf in workflows:
if wf not in existing:
print "putting",wf
new_wf = Workflow( name = wf , status = options.setstatus, wm_status = options.wmstatus)
session.add( new_wf )
session.commit()
existing = [wf.name for wf in session.query(Workflow).all()]
## pick up replacements
for wf in session.query(Workflow).filter(Workflow.status == 'trouble').all():
if specific and wf.name != specific:
continue
print wf.name
wl = getWorkLoad(url, wf.name)
familly = getWorkflowById( url, wl['PrepID'] )
if len(familly)==1:
print wf.name,"ERROR has no replacement"
continue
print wf.name,"has",len(familly),"familly members"
for member in familly:
if member != wf.name:
fwl = getWorkLoad(url , member)
if options.replace:
if member != options.replace: continue
else:
if fwl['RequestDate'] < wl['RequestDate']: continue
if fwl['RequestType']=='Resubmission': continue
if fwl['RequestStatus'] in ['None',None]: continue
new_wf = session.query(Workflow).filter(Workflow.name == member).first()
if not new_wf:
print "putting",member
status = 'away'
if fwl['RequestStatus'] in ['assignment-approved']:
status = 'considered'
new_wf = Workflow( name = member, status = status, wm_status = fwl['RequestStatus'])
wf.status = 'forget'
session.add( new_wf )
session.commit()
else:
if new_wf.status == 'forget': continue
print "getting",new_wf.name,"as replacement of",wf.name
for tr in session.query(Transfer).all():
if wf.id in tr.workflows_id:
sw = copy.deepcopy(tr.workflows_id)
sw.remove( wf.id)
sw.append(new_wf.id)
tr.workflows_id = sw
print tr.phedexid,"got",new_wf.name
if new_wf.status != 'away':
new_wf.status = 'staging'
session.commit()
## don't do that automatically
#wf.status = 'forget'
session.commit()
def closor(url, specific=None):
dbsapi = DbsApi(url='https://cmsweb.cern.ch/dbs/prod/global/DBSReader')
for wfo in session.query(Workflow).filter(Workflow.status=='away').all():
if specific and not specific in wfo.name: continue
## what is the expected #lumis
wl = getWorkLoad(url, wfo.name)
if wfo.wm_status != wl['RequestStatus']:
wfo.wm_status = wl['RequestStatus']
session.commit()
take_out = ['failed','aborted','aborted-archived','rejected','rejected-archived']
if wl['RequestStatus'] in take_out:
wfo.status = 'trouble'
wfo.wm_status = wl['RequestStatus']
session.commit()
continue
if wl['RequestStatus'] in ['announced','normal-archived']:
wfo.status = 'done'
wfo.wm_status = wl['RequestStatus']
session.commit()
continue
if wl['RequestType'] == 'Resubmission':
#session.delete( wl)
#session.commit()
print wfo.name,"can be taken out"
wfo.status = 'forget'
wfo.wm_status = wl['RequestStatus']
session.commit()
continue
if wl['RequestStatus'] in ['assigned','acquired']:
print wfo.name,"not running yet"
continue
if not 'TotalInputLumis' in wl:
print wfo.name,"has not been assigned yet"
continue
expected_lumis = wl['TotalInputLumis']
## what are the outputs
outputs = wl['OutputDatasets']
## check whether the number of lumis is as expected for each
all_OK = []
#print outputs
if len(outputs):
print wfo.name,wl['RequestStatus']
for out in outputs:
reply = dbsapi.listFileSummaries(dataset=out)
lumi_count = 0
event_count = 0
for f in reply:
lumi_count +=f['num_lumi']
event_count +=f['num_event']
odb = session.query(Output).filter(Output.datasetname==out).first()
if not odb:
print "adding an output object",out
odb = Output( datasetname = out )
odb.workflow = wfo
session.add( odb )
odb.nlumis = lumi_count
odb.nevents = event_count
if odb.expectedlumis < expected_lumis:
odb.expectedlumis = expected_lumis
else:
expected_lumis = odb.expectedlumis
odb.date = time.mktime(time.gmtime())
session.commit()
print "\t%60s %d/%d = %3.2f%%"%(out,lumi_count,expected_lumis,lumi_count/float(expected_lumis)*100.)
#print wfo.fraction_for_closing, lumi_count, expected_lumis
fraction = wfo.fraction_for_closing
fraction = 0.05
all_OK.append((float(lumi_count) > float(expected_lumis*fraction)))
## only that status can let me go into announced
if wl['RequestStatus'] in ['closed-out']: ## add force-completed ??
print wfo.name,"to be announced"
results=[]#'dummy']
if not results:
results.append(reqMgrClient.announceWorkflowCascade(url, wfo.name))
for (io,out) in enumerate(outputs):
if all_OK[io]:
results.append(setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', out, 'VALID' ,''))
## inject to DDM everything from ReDigi
if wl['RequestType'] == 'ReDigi' and not ('/DQM' in out):
print "Sending",out," to DDM"
subprocess.call(['python','assignDatasetToSite.py','--dataset='+out,'--exec'])
else:
print wfo.name,"no stats for announcing",out
results.append(None)
#print results
if all(map(lambda result : result in ['None',None],results)):
wfo.status = 'done'
session.commit()
print wfo.name,"is announced"
else:
print "ERROR with ",wfo.name,"to be announced",json.dumps( results )
else:
print wfo.name,"not good for announcing:",wl['RequestStatus']
def injector(url, options, specific):
use_mcm = True
up = componentInfo( mcm = use_mcm, soft=['mcm'] )
if not up.check(): return
use_mcm = up.status['mcm']
workflows = getWorkflows(url, status=options.wmstatus, user=options.user)
workflows.extend( getWorkflows(url, status=options.wmstatus, user='******', rtype="ReReco")) ## regardless of users, pick up all ReReco on the table
existing = [wf.name for wf in session.query(Workflow).all()]
## browse for assignment-approved requests, browsed for ours, insert the diff
for wf in workflows:
if wf not in existing:
print "putting",wf
new_wf = Workflow( name = wf , status = options.setstatus, wm_status = options.wmstatus)
session.add( new_wf )
session.commit()
time.sleep(1)
existing = [wf.name for wf in session.query(Workflow).all()]
## passing a round of invalidation of what needs to be invalidated
if use_mcm and (options.invalidate or True):
invalidator(url)
## pick up replacements
for wf in session.query(Workflow).filter(Workflow.status == 'trouble').all():
if specific and wf.name != specific:
continue
print wf.name
wl = getWorkLoad(url, wf.name)
familly = getWorkflowById( url, wl['PrepID'] )
true_familly = []
for member in familly:
if member == wf.name: continue
fwl = getWorkLoad(url , member)
if options.replace:
if member != options.replace: continue
else:
if fwl['RequestDate'] < wl['RequestDate']: continue
if fwl['RequestType']=='Resubmission': continue
if fwl['RequestStatus'] in ['None',None]: continue
true_familly.append( fwl )
if len(true_familly)==0:
print wf.name,"ERROR has no replacement"
known = []
try:
known = json.loads(open('no_replacement.json').read())
except:
pass
if not wf.name in known:
sendEmail('workflow in %s with no replacement'%(wl['RequestStatus']),'%s is dangling there'%(wf.name))
known.append( wf.name )
open('no_replacement.json','w').write( json.dumps( known, indent=2 ))
continue
print wf.name,"has",len(familly),"familly members"
print wf.name,"has",len(true_familly),"true familly members"
for fwl in true_familly:
member = fwl['RequestName']
new_wf = session.query(Workflow).filter(Workflow.name == member).first()
if not new_wf:
print "putting",member,"as replacement of",wf.name
status = 'away'
if fwl['RequestStatus'] in ['assignment-approved']:
status = 'considered'
new_wf = Workflow( name = member, status = status, wm_status = fwl['RequestStatus'])
wf.status = 'forget'
session.add( new_wf )
else:
if new_wf.status == 'forget': continue
print "getting",new_wf.name,"as replacement of",wf.name
wf.status = 'forget'
for tr in session.query(Transfer).all():
if wf.id in tr.workflows_id:
sw = copy.deepcopy(tr.workflows_id)
sw.remove( wf.id)
sw.append(new_wf.id)
tr.workflows_id = sw
print tr.phedexid,"got",new_wf.name
if new_wf.status != 'away':
print "\t setting it considered"
new_wf.status = 'considered'
if tr.phedexid<0: ## set it back to positive
tr.phedexid = -tr.phedexid
session.commit()
## don't do that automatically
#wf.status = 'forget'
session.commit()
def injector(url, options, specific):
mlock = moduleLock()
if mlock(): return
use_mcm = True
up = componentInfo(soft=['mcm', 'wtc'])
if not up.check(): return
use_mcm = up.status['mcm']
UC = unifiedConfiguration()
transform_keywords = UC.get('convert_to_stepchain')
workflows = getWorkflows(url, status=options.wmstatus, user=options.user)
for user in UC.get("user_rereco"):
workflows.extend(
getWorkflows(url,
status=options.wmstatus,
user=user,
rtype="ReReco"))
for user in (options.user_relval.split(',')
if options.user_relval else UC.get("user_relval")):
workflows.extend(
getWorkflows(url,
status=options.wmstatus,
user=user,
rtype="TaskChain"))
for user in (options.user_storeresults.split(',') if
options.user_storeresults else UC.get("user_storeresults")):
workflows.extend(
getWorkflows(url,
status=options.wmstatus,
user=user,
rtype="StoreResults"))
print len(workflows), "in line"
cannot_inject = set()
to_convert = set()
status_cache = defaultdict(str)
## browse for assignment-approved requests, browsed for ours, insert the diff
for wf in workflows:
if specific and not specific in wf: continue
exists = session.query(Workflow).filter(Workflow.name == wf).first()
if not exists:
wfi = workflowInfo(url, wf)
## check first that there isn't related here with something valid
can_add = True
## first try at finding a match
familly = session.query(Workflow).filter(
Workflow.name.contains(wfi.request['PrepID'])).all()
if not familly:
pids = wfi.getPrepIDs()
req_familly = []
for pid in pids:
req_familly.extend(getWorkflowById(url, pid, details=True))
familly = []
print len(req_familly), "members"
for req_member in req_familly:
#print "member",req_member['RequestName']
owfi = workflowInfo(url,
req_member['RequestName'],
request=req_member)
other_pids = owfi.getPrepIDs()
if set(pids) == set(other_pids):
## this is a real match
familly.extend(
session.query(Workflow).filter(
Workflow.name ==
req_member['RequestName']).all())
for lwfo in familly:
if lwfo:
## we have it already
if not lwfo.status in [
'forget', 'trouble', 'forget-unlock',
'forget-out-unlock'
]:
wfi.sendLog(
'injector', "Should not put %s because of %s %s" %
(wf, lwfo.name, lwfo.status))
sendLog('injector',
"Should not put %s because of %s %s" %
(wf, lwfo.name, lwfo.status),
level='critical')
print "Should not put", wf, "because of", lwfo.name, lwfo.status
cannot_inject.add(wf)
can_add = False
## add a check on validity of input datasets
_, prim, par, sec = wfi.getIO()
for d in list(prim) + list(par) + list(sec):
if not d in status_cache:
status_cache[d] = getDatasetStatus(d)
if status_cache[d] != 'VALID':
wfi.sendLog(
'injector', "One of the input is not VALID. %s : %s" %
(d, status_cache[d]))
sendLog('injector',
"One of the input of %s is not VALID. %s : %s" %
(wf, d, status_cache[d]),
level='critical')
can_add = False
## check for any file in phedex, to verify existence
_, ph_files, _, _ = getDatasetFiles(url, d)
if not ph_files and not ('StoreResults'
== wfi.request.setdefault(
'RequestType', None)):
wfi.sendLog(
'injector',
"One of the input has no file in phedex: %s" % d)
sendLog('injector',
"One of the input has no file in phedex: %s" % d,
level='critical')
can_add = False
### ban some workflow that you don't like anymore
#outputs = wfi.request['OutputDatasets']
if not can_add: continue
## temporary hack to transform specific taskchain into stepchains
#good_for_stepchain = wfi.isGoodToConvertToStepChain( keywords = transform_keywords)
good_for_stepchain = wfi.isGoodToConvertToStepChain(keywords=None)
## match keywords and technical constraints
#if (not options.no_convert) and good_for_stepchain and not wfi.isRelval():
# to_convert.add( wf )
# wfi.sendLog('injector','Transforming %s TaskChain into StepChain'%wf)
# #sendEmail('convertion to stepchain','Transforming %s TaskChain into StepChain'%wf)
wfi.sendLog('injector', "considering %s" % wf)
new_wf = Workflow(name=wf,
status=options.setstatus,
wm_status=options.wmstatus)
session.add(new_wf)
session.commit()
time.sleep(0.5)
else:
#print "already have",wf
pass
if cannot_inject:
#sendEmail('workflow duplicates','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject)))
sendLog(
'injector',
'These workflow cannot be added in because of duplicates \n\n %s' %
('\n'.join(cannot_inject)),
level='warning')
for wf in to_convert:
os.system(
'./Unified/rejector.py --clone --to_step --comments \"Transform to StepChain\" %s'
% wf)
## passing a round of invalidation of what needs to be invalidated
if use_mcm and (options.invalidate or True):
invalidator(url)
no_replacement = set()
#print "getting all transfers"
#all_transfers=session.query(Transfer).all()
#print "go!"
## pick up replacements
for wf in session.query(Workflow).filter(
Workflow.status == 'trouble').all():
print wf.name
if specific and not specific in wf.name: continue
print wf.name
wfi = workflowInfo(url, wf.name)
wl = wfi.request #getWorkLoad(url, wf.name)
familly = getWorkflowById(url, wl['PrepID'])
true_familly = []
for member in familly:
if member == wf.name: continue
fwl = getWorkLoad(url, member)
if options.replace:
if member != options.replace: continue
else:
if fwl['RequestDate'] < wl['RequestDate']: continue
if fwl['RequestType'] == 'Resubmission': continue
if fwl['RequestStatus'] in ['None', None, 'new']: continue
if fwl['RequestStatus'] in [
'rejected', 'rejected-archived', 'aborted',
'aborted-archived'
]:
continue
true_familly.append(fwl)
if len(true_familly) == 0:
#sendLog('injector','%s had no replacement'%wf.name, level='critical')
if wfi.isRelval():
#wfi.sendLog('injector','the workflow was found in trouble with no replacement. As a relval, there is no clean way to handle this.')
wfi.sendLog(
'injector',
'the workflow was found in trouble with no replacement. As a relval, there is no clean way to handle this. Setting forget'
)
wf.status = 'forget'
session.commit()
else:
wfi.sendLog(
'injector',
'the workflow was found in trouble with no replacement')
no_replacement.add(wf.name)
continue
else:
wfi.sendLog(
'injector',
'the workflow was found in trouble and has a replacement')
print wf.name, "has", len(familly), "familly members"
print wf.name, "has", len(true_familly), "true familly members"
##we cannot have more than one of them !!! pick the last one
if len(true_familly) > 1:
#sendEmail('multiple wf','please take a look at injector for %s'%wf.name)
sendLog('injector',
'Multiple wf in line, will take the last one for %s \n%s' %
(wf.name, ', '.join(fwl['RequestName']
for fwl in true_familly)),
level='critical')
for fwl in true_familly[-1:]:
member = fwl['RequestName']
new_wf = session.query(Workflow).filter(
Workflow.name == member).first()
if not new_wf:
sendLog('injector',
"putting %s as replacement of %s" % (member, wf.name))
status = 'away'
if fwl['RequestStatus'] in ['assignment-approved']:
status = 'considered'
new_wf = Workflow(name=member,
status=status,
wm_status=fwl['RequestStatus'])
wf.status = 'forget'
session.add(new_wf)
else:
if new_wf.status == 'forget': continue
sendLog(
'injector',
"getting %s as replacement of %s" % (new_wf.name, wf.name))
wf.status = 'forget'
for tr in session.query(TransferImp).filter(
TransferImp.workflow_id == wf.id).all():
## get all transfer working for the old workflow
existing = session.query(TransferImp).filter(
TransferImp.phedexid == tr.phedexid).filter(
TransferImp.workflow_id == new_wf.id).all()
tr.active = False ## disable the old one
if not existing:
## create the transfer object for the new dependency
tri = TransferImp(phedexid=tr.phedexid, workflow=new_wf)
session.add(tri)
session.commit()
## don't do that automatically
#wf.status = 'forget'
session.commit()
if no_replacement:
#sendEmail('workflow with no replacement','%s \n are dangling there'%( '\n'.join(no_replacement)))
sendLog('injector',
'workflow with no replacement\n%s \n are dangling there' %
('\n'.join(no_replacement)),
level='critical')
def wfl(wf,
view=False,
p=False,
ms=False,
within=False,
ongoing=False,
status=False,
update=False):
wfn = wf.name
wfs = wf.wm_status
pid = None
pids = filter(lambda seg: seg.count('-') == 2, wf.name.split('_'))
if len(pids):
pid = pids[0]
text = ', '.join([
#wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a>'
% (wfn, wfn),
'(%s) <br>' % wfs
])
text += ', '.join([
'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts</a>'
% wfn,
'<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>'
% wfn,
'<a href="https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>'
% wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>'
% wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'
% wfn,
'<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>'
% wfn,
'<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>'
% wfn,
'<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'
% pid,
'<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank">pv</a>'
% wfn,
'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>'
% wfn,
'<a href="closeout.html#%s" target="_blank">clo</a>' % wfn,
'<a href="statuses.html#%s" target="_blank">st</a>' % wfn,
'<a href="https://cmsweb.cern.ch/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>'
% wfn
])
if within and (not view or wfs == 'completed'):
cached = filter(lambda d: d['RequestName'] == wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad('cmsweb.cern.ch', wfn)
if 'InputDataset' in wl:
dataset = wl['InputDataset']
text += ', '.join([
'',
'<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>'
% dataset,
'<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>'
% dataset,
'<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>'
% dataset,
'<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>'
% dataset,
])
if p:
cached = filter(lambda d: d['RequestName'] == wfn, cache)
if cached:
wl = cached[0]
else:
wl = getWorkLoad('cmsweb.cern.ch', wfn)
text += ', (%s)' % (wl['RequestPriority'])
pass
if pid:
if ms:
mcm_s = json.loads(
os.popen(
'curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure'
% pid).read())[pid]
text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>' % (
pid, mcm_s)
else:
text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>' % (
pid)
text += ', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>' % (
pid)
if status:
if wf.status.startswith('assistance'):
text += ', <a href="assistance.html#%s" target="_blank">assist</a>' % wfn
text += ' : %s ' % (wf.status)
if view and wfs != 'acquired':
text += '<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>' % (
wfn.replace('_', '/'), wfn.replace('_', '/'))
if ongoing:
text += '<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank"><img src="http://hcc-briantest.unl.edu/prodview/graphs/%s/daily" style="height:50px"></a>' % (
wfn, wfn)
text += "<hr>"
return text
