def wfl(wf,v=False,p=False,ms=False): wfn = wf.name wfs = wf.wm_status pid = None pids=filter(lambda seg: seg.count('-')==2, wf.name.split('_')) if len(pids): pid=pids[0] text=', '.join([ #wfn, '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a>'%(wfn,wfn), '(%s)'%wfs, '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts</a>'%wfn, '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">wkl</a>'%wfn, '<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'%wfn, '<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>'%wfn, '<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'%pid, '<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank">pv</a>'%wfn ]) if p: wl = getWorkLoad('cmsweb.cern.ch',wfn) text+=', (%s)'%(wl['RequestPriority']) if pid: if ms: mcm_s = json.loads(os.popen('curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure'%pid).read())[pid] text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>'%(pid,mcm_s) else: text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>'%(pid) if v and wfs!='acquired': text+='<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>'%(wfn.replace('_','/'),wfn.replace('_','/')) text+='<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank"><img src="http://hcc-briantest.unl.edu/prodview/graphs/%s/daily" style="height:50px"></a>'%(wfn,wfn) return text
def getWL( wfn ): cached = filter(lambda d : d['RequestName']==wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad(reqmgr_url,wfn) return wl
def rejector(url, specific ): if specific.startswith('/'): pass else: wfo = session.query(Workflow).filter(Workflow.name == specific).first() if not wfo: print "cannot reject",spec return results=[] wl = getWorkLoad(url, wfo.name) if wl['RequestStatus'] in ['assignment-approved','new']: #results.append( reqMgrClient.rejectWorkflow(url, wfo.name)) reqMgrClient.rejectWorkflow(url, wfo.name) else: #results.append( reqMgrClient.abortWorkflow(url, wfo.name)) reqMgrClient.abortWorkflow(url, wfo.name) datasets = wl['OutputDatasets'] for dataset in datasets: results.append( setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', dataset, 'INVALID', None) ) if all(map(lambda result : result in ['None',None],results)): wfo.status = 'forget' session.commit() print wfo.name,"and",datasets,"are rejected" else: print "error in rejecting",wfo.name,results
def getWL(wfn): cached = filter(lambda d: d['RequestName'] == wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad(reqmgr_url, wfn) return wl
def getWL(wfn): cached = filter(lambda d: d["RequestName"] == wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad("cmsweb.cern.ch", wfn) return wl
def completor(url, specific): wfs = [] wfs.extend( session.query(Workflow).filter(Workflow.status == 'away').all() ) wfs.extend( session.query(Workflow).filter(Workflow.status.startswith('assistance')).all() ) for wfo in wfs: if specific and not specific in wfo.name: continue ## get all of the same wl = getWorkLoad(url, wfo.name) familly = getWorkflowById( url, wl['PrepID'] ,details=True) for member in familly: ### if member['RequestName'] == wl['RequestName']: continue if member['RequestDate'] < wl['RequestDate']: continue if member['RequestStatus'] in ['None',None]: continue ## then set force complete all members if member['RequestStatus'] in ['running-opened','running-closed']: print "setting",member['RequestName'],"force-complete" reqMgrClient.setWorkflowForceComplete(url, member['RequestName'])
def completor(url, specific): wfs = [] wfs.extend(session.query(Workflow).filter(Workflow.status == "away").all()) wfs.extend(session.query(Workflow).filter(Workflow.status.startswith("assistance")).all()) for wfo in wfs: if specific and not specific in wfo.name: continue ## get all of the same wl = getWorkLoad(url, wfo.name) familly = getWorkflowById(url, wl["PrepID"], details=True) for member in familly: ### if member['RequestName'] == wl['RequestName']: continue if member["RequestDate"] < wl["RequestDate"]: continue if member["RequestStatus"] in ["None", None]: continue ## then set force complete all members if member["RequestStatus"] in ["running-opened", "running-closed"]: print "setting", member["RequestName"], "force-complete" reqMgrClient.setWorkflowForceComplete(url, member["RequestName"])
def wfl(wf,view=False,p=False,ms=False,within=False,ongoing=False,status=False,update=False): wfn = wf.name wfs = wf.wm_status wl = None pid = None wl_pid = None pids=filter(lambda seg: seg.count('-')==2, wf.name.split('_')) if len(pids): pids = pids[:1] pid=pids[0] if not pids: wl = getWL( wf.name ) pids = getPrepIDs( wl ) pid = pids[0] wl_pid = pid if 'task' in wf.name: wl_pid = 'task_'+pid text=', '.join([ #wfn, #'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a> '%(wfn,wfn), #'<table><tr><td>%s</td></tr></table>'%(wfn), #'<span>%s</span>'%(wfn), "%s "%wfn, '(%s) <br>'%wfs]) text+=', '.join([ '<a href="https://%s/reqmgr2/fetch?rid=%s" target="_blank">dts</a>'%(reqmgr_url,wfn), '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts-req1</a>'%wfn, #TOFIX '<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>'%wfn, '<a href="https://%s/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>'%(reqmgr_url,wfn), '<a href="https://%s/reqmgr2/data/request?name=%s" target="_blank">req</a>'%(reqmgr_url,wfn), #'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>'%wfn, #TOFIX '<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'%wfn, '<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>'%wfn, '<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>'%wfn, '<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'%pid, '<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank">pv</a>'%wfn, #deprecated '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>'%wfn, '<a href="closeout.html#%s" target="_blank">clo</a>'%wfn, '<a href="statuses.html#%s" target="_blank">st</a>'%wfn, '<a href="https://%s/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>'%(reqmgr_url,wfn) ]) if within and (not view or wfs=='completed'): wl = getWL( wfn ) dataset =None if 'InputDataset' in wl: dataset = wl['InputDataset'] if 'Task1' in wl and 'InputDataset' in wl['Task1']: dataset = wl['Task1']['InputDataset'] if dataset: text+=', '.join(['', '<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>'%dataset, '<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>'%dataset, '<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>'%dataset, '<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>'%dataset, ]) if p: cached = filter(lambda d : d['RequestName']==wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad('cmsweb.cern.ch',wfn) text+=', (%s)'%(wl['RequestPriority']) pass if pid: if ms: mcm_s = json.loads(os.popen('curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure'%pid).read())[pid] text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>'%(pid,mcm_s) else: text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>'%(pid) text+=', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>'%(wl_pid) if status: if wf.status.startswith('assistance'): text+=', <a href="assistance.html#%s" target="_blank">assist</a>'%wfn text+=' : %s '%(wf.status) if view and wfs!='acquired': text+='<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>'%(wfn.replace('_','/'),wfn.replace('_','/')) if ongoing: text+='<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank"><img src="http://cms-gwmsmon.cern.ch/prodview/graphs/%s/daily" style="height:50px"></a>'%(wfn,wfn) if ongoing: date1 = time.strftime('%Y-%m-%d+%H:%M', time.gmtime(time.mktime(time.gmtime())-(15*24*60*60)) ) date2 = time.strftime('%Y-%m-%d+%H:%M', time.gmtime()) text+='<a href="http://dashb-cms-job.cern.ch/dashboard/templates/web-job2/#table=Jobs&date1=%s&date2=%s&sortby=site&task=wmagent_%s">dashb</a>'%( date1, date2, wfn ) if ongoing and wfn in boost: for task in boost[wfn]: overflow = boost[wfn][task].get('ReplaceSiteWhitelist',None) if not overflow: overflow = boost[wfn][task].get('AddWhitelist',None) if overflow: text+=',boost (<a href=equalizor.json>%d</a>)'%len(overflow) #text+="<hr>" return text
def injector(url, options, specific): ## passing a round of invalidation of what needs to be invalidated if options.invalidate: invalidator(url) workflows = getWorkflows(url, status=options.wmstatus, user=options.user) existing = [wf.name for wf in session.query(Workflow).all()] ## browse for assignment-approved requests, browsed for ours, insert the diff for wf in workflows: if wf not in existing: print "putting", wf new_wf = Workflow(name=wf, status=options.setstatus, wm_status=options.wmstatus) session.add(new_wf) session.commit() existing = [wf.name for wf in session.query(Workflow).all()] ## pick up replacements for wf in session.query(Workflow).filter( Workflow.status == 'trouble').all(): if specific and wf.name != specific: continue print wf.name wl = getWorkLoad(url, wf.name) familly = getWorkflowById(url, wl['PrepID']) if len(familly) == 1: print wf.name, "ERROR has no replacement" continue print wf.name, "has", len(familly), "familly members" for member in familly: if member != wf.name: fwl = getWorkLoad(url, member) if options.replace: if member != options.replace: continue else: if fwl['RequestDate'] < wl['RequestDate']: continue if fwl['RequestType'] == 'Resubmission': continue if fwl['RequestStatus'] in ['None', None]: continue new_wf = session.query(Workflow).filter( Workflow.name == member).first() if not new_wf: print "putting", member status = 'away' if fwl['RequestStatus'] in ['assignment-approved']: status = 'considered' new_wf = Workflow(name=member, status=status, wm_status=fwl['RequestStatus']) wf.status = 'forget' session.add(new_wf) session.commit() else: if new_wf.status == 'forget': continue print "getting", new_wf.name, "as replacement of", wf.name for tr in session.query(Transfer).all(): if wf.id in tr.workflows_id: sw = copy.deepcopy(tr.workflows_id) sw.remove(wf.id) sw.append(new_wf.id) tr.workflows_id = sw print tr.phedexid, "got", new_wf.name if new_wf.status != 'away': new_wf.status = 'staging' session.commit() ## don't do that automatically #wf.status = 'forget' session.commit()
wfs = [] for status in statuses: wfs.extend( session.query(Workflow).filter(Workflow.status==status).all() ) if max_wf: wfs = wfs[:max_wf] random.shuffle( wfs ) all_blocks_at_sites = defaultdict(set) done = json.loads(open('myblock_done.json').read()) print len(wfs),"to look the output of" for iw,wfo in enumerate(wfs): print "%s/%s:"%(iw,len(wfs)),wfo.name #wfi = workflowInfo(url, wfo.name) #outs= wfi.request['OutputDatasets'] wl = getWorkLoad(url, wfo.name) outs= wl['OutputDatasets'] for out in outs: blocks_at_sites = getDatasetBlockAndSite(url, out, group="") deletions = getDatasetOnGoingDeletion(url, out) if len(deletions): print "\t\tshould not subscribe with on-going deletions",out continue for site,blocks in blocks_at_sites.items(): if 'Buffer' in site or 'Export' in site or 'MSS' in site: continue all_blocks_at_sites[site].update( blocks ) print "\t",out print "\t\t",len(blocks_at_sites),"sites",sorted(blocks_at_sites.keys()),"with unsubscribed blocks" if len(all_blocks_at_sites.keys())==0 and len(wfs): ## no subscription to be done at this time, let me know
def injector(url, options, specific): use_mcm = True up = componentInfo( mcm = use_mcm, soft=['mcm'] ) if not up.check(): return use_mcm = up.status['mcm'] workflows = getWorkflows(url, status=options.wmstatus, user=options.user) workflows.extend( getWorkflows(url, status=options.wmstatus, user='******', rtype="ReReco")) ## regardless of users, pick up all ReReco on the table print len(workflows),"in line" cannot_inject = set() status_cache = defaultdict(str) ## browse for assignment-approved requests, browsed for ours, insert the diff for wf in workflows: if specific and not specific in wf: continue exists = session.query(Workflow).filter(Workflow.name == wf ).first() if not exists: wfi = workflowInfo(url, wf) #wl = getWorkLoad(url, wf) ## check first that there isn't related here with something valid can_add = True ## first try at finding a match # print wfi.request familly = session.query(Workflow).filter(Workflow.name.contains(wfi.request['PrepID'])).all() if not familly: #req_familly = getWorkflowById( url, wl['PrepID']) #familly = [session.query(Workflow).filter(Workflow.name == member).first() for member in req_familly] pids = wfi.getPrepIDs() req_familly = [] for pid in pids: req_familly.extend( getWorkflowById( url, pid, details=True) ) familly = [] print len(req_familly),"members" for req_member in req_familly: #print "member",req_member['RequestName'] owfi = workflowInfo(url, req_member['RequestName'], request=req_member) other_pids = owfi.getPrepIDs() if set(pids) == set(other_pids): ## this is a real match familly.extend( session.query(Workflow).filter(Workflow.name == req_member['RequestName']).all() ) for lwfo in familly: if lwfo: ## we have it already if not lwfo.status in ['forget','trouble','forget-unlock','forget-out-unlock']: sendLog('injector',"Should not put %s because of %s %s"%( wf, lwfo.name,lwfo.status )) print "Should not put",wf,"because of",lwfo.name,lwfo.status cannot_inject.add( wf ) can_add = False ## add a check on validity of input datasets _,prim,par,sec = wfi.getIO() for d in list(prim)+list(par)+list(sec): if not d in status_cache: status_cache[d] = getDatasetStatus(d) if status_cache[d] != 'VALID': wfi.sendLog('injector',"One of the input is not VALID. %s : %s"%( d, status_cache[d])) sendLog('injector',"One of the input of %s is not VALID. %s : %s"%( wf, d, status_cache[d])) can_add = False if not can_add: continue wfi.sendLog('injector',"considering %s"%wf) new_wf = Workflow( name = wf , status = options.setstatus, wm_status = options.wmstatus) session.add( new_wf ) session.commit() time.sleep(0.5) else: #print "already have",wf pass if cannot_inject: #sendEmail('workflow duplicates','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject))) sendLog('injector','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject)), level='warning') ## passing a round of invalidation of what needs to be invalidated if use_mcm and (options.invalidate or True): invalidator(url) no_replacement = set() ## pick up replacements for wf in session.query(Workflow).filter(Workflow.status == 'trouble').all(): print wf.name if specific and not specific in wf.name: continue print wf.name wfi = workflowInfo(url, wf.name ) wl = wfi.request #getWorkLoad(url, wf.name) familly = getWorkflowById( url, wl['PrepID'] ) true_familly = [] for member in familly: if member == wf.name: continue fwl = getWorkLoad(url , member) if options.replace: if member != options.replace: continue else: if fwl['RequestDate'] < wl['RequestDate']: continue if fwl['RequestType']=='Resubmission': continue if fwl['RequestStatus'] in ['None',None,'new']: continue if fwl['RequestStatus'] in ['rejected','rejected-archived','aborted','aborted-archived']: continue true_familly.append( fwl ) if len(true_familly)==0: #sendLog('injector','%s had no replacement'%wf.name, level='critical') wfi.sendLog('injector','the workflow was found in trouble with no replacement') no_replacement.add( wf.name ) continue else: wfi.sendLog('injector','the workflow was found in trouble and has a replacement') print wf.name,"has",len(familly),"familly members" print wf.name,"has",len(true_familly),"true familly members" ##we cannot have more than one of them !!! pick the last one if len(true_familly)>1: #sendEmail('multiple wf','please take a look at injector for %s'%wf.name) sendLog('injector','Multiple wf in line, will take the last one for %s \n%s'%( wf.name, ', '.join(fwl['RequestName'] for fwl in true_familly)), level='critical') for fwl in true_familly[-1:]: member = fwl['RequestName'] new_wf = session.query(Workflow).filter(Workflow.name == member).first() if not new_wf: sendLog('injector',"putting %s as replacement of %s"%( member, wf.name)) status = 'away' if fwl['RequestStatus'] in ['assignment-approved']: status = 'considered' new_wf = Workflow( name = member, status = status, wm_status = fwl['RequestStatus']) wf.status = 'forget' session.add( new_wf ) else: if new_wf.status == 'forget': continue sendLog('injector',"getting %s as replacement of %s"%( new_wf.name, wf.name )) wf.status = 'forget' for tr in session.query(Transfer).all(): if wf.id in tr.workflows_id: sw = copy.deepcopy(tr.workflows_id) sw.remove( wf.id) sw.append(new_wf.id) tr.workflows_id = sw print tr.phedexid,"got",new_wf.name if new_wf.status != 'away': print "\t setting it considered" new_wf.status = 'considered' if tr.phedexid<0: ## set it back to positive tr.phedexid = -tr.phedexid session.commit() ## don't do that automatically #wf.status = 'forget' session.commit() if no_replacement: #sendEmail('workflow with no replacement','%s \n are dangling there'%( '\n'.join(no_replacement))) sendLog('injector','workflow with no replacement, %s \n are dangling there'% ( '\n'.join(no_replacement)), level='critical')
def injector(url, options, specific): use_mcm = True up = componentInfo(mcm=use_mcm, soft=["mcm"]) if not up.check(): return use_mcm = up.status["mcm"] workflows = getWorkflows(url, status=options.wmstatus, user=options.user) existing = [wf.name for wf in session.query(Workflow).all()] ## browse for assignment-approved requests, browsed for ours, insert the diff for wf in workflows: if wf not in existing: print "putting", wf new_wf = Workflow(name=wf, status=options.setstatus, wm_status=options.wmstatus) session.add(new_wf) session.commit() time.sleep(1) existing = [wf.name for wf in session.query(Workflow).all()] ## passing a round of invalidation of what needs to be invalidated if use_mcm and (options.invalidate or True): invalidator(url) ## pick up replacements for wf in session.query(Workflow).filter(Workflow.status == "trouble").all(): if specific and wf.name != specific: continue print wf.name wl = getWorkLoad(url, wf.name) familly = getWorkflowById(url, wl["PrepID"]) true_familly = [] for member in familly: if member == wf.name: continue fwl = getWorkLoad(url, member) if options.replace: if member != options.replace: continue else: if fwl["RequestDate"] < wl["RequestDate"]: continue if fwl["RequestType"] == "Resubmission": continue if fwl["RequestStatus"] in ["None", None]: continue true_familly.append(fwl) if len(true_familly) == 0: print wf.name, "ERROR has no replacement" known = [] try: known = json.loads(open("no_replacement.json").read()) except: pass if not wf.name in known: sendEmail( "workflow in %s with no replacement" % (wl["RequestStatus"]), "%s is dangling there" % (wf.name) ) known.append(wf.name) open("no_replacement.json", "w").write(json.dumps(known, indent=2)) continue print wf.name, "has", len(familly), "familly members" print wf.name, "has", len(true_familly), "true familly members" for fwl in true_familly: member = fwl["RequestName"] new_wf = session.query(Workflow).filter(Workflow.name == member).first() if not new_wf: print "putting", member, "as replacement of", wf.name status = "away" if fwl["RequestStatus"] in ["assignment-approved"]: status = "considered" new_wf = Workflow(name=member, status=status, wm_status=fwl["RequestStatus"]) wf.status = "forget" session.add(new_wf) else: if new_wf.status == "forget": continue print "getting", new_wf.name, "as replacement of", wf.name wf.status = "forget" for tr in session.query(Transfer).all(): if wf.id in tr.workflows_id: sw = copy.deepcopy(tr.workflows_id) sw.remove(wf.id) sw.append(new_wf.id) tr.workflows_id = sw print tr.phedexid, "got", new_wf.name if new_wf.status != "away": print "\t setting it considered" new_wf.status = "considered" session.commit() ## don't do that automatically # wf.status = 'forget' session.commit()
def closor(url, specific=None): if not componentInfo().check(): return CI = campaignInfo() LI = lockInfo() ## manually closed-out workflows should get to close with checkor for wfo in session.query(Workflow).filter(Workflow.status=='close').all(): if specific and not specific in wfo.name: continue ## what is the expected #lumis wl = getWorkLoad(url, wfo.name) wfo.wm_status = wl['RequestStatus'] if wl['RequestStatus'] in ['announced','normal-archived']: ## manually announced ?? wfo.status = 'done' wfo.wm_status = wl['RequestStatus'] print wfo.name,"is announced already",wfo.wm_status session.commit() expected_lumis = 1 if not 'TotalInputLumis' in wl: print wfo.name,"has not been assigned yet, or the database is corrupted" else: expected_lumis = wl['TotalInputLumis'] ## what are the outputs outputs = wl['OutputDatasets'] ## check whether the number of lumis is as expected for each all_OK = [] #print outputs if len(outputs): print wfo.name,wl['RequestStatus'] for out in outputs: event_count,lumi_count = getDatasetEventsAndLumis(dataset=out) odb = session.query(Output).filter(Output.datasetname==out).first() if not odb: print "adding an output object",out odb = Output( datasetname = out ) odb.workflow = wfo session.add( odb ) odb.nlumis = lumi_count odb.nevents = event_count odb.workfow_id = wfo.id if odb.expectedlumis < expected_lumis: odb.expectedlumis = expected_lumis else: expected_lumis = odb.expectedlumis odb.date = time.mktime(time.gmtime()) session.commit() print "\t%60s %d/%d = %3.2f%%"%(out,lumi_count,expected_lumis,lumi_count/float(expected_lumis)*100.) #print wfo.fraction_for_closing, lumi_count, expected_lumis fraction = wfo.fraction_for_closing fraction = 0.0 all_OK.append((float(lumi_count) > float(expected_lumis*fraction))) ## only that status can let me go into announced if wl['RequestStatus'] in ['closed-out']: print wfo.name,"to be announced" results=[]#'dummy'] if not results: for (io,out) in enumerate(outputs): if all_OK[io]: results.append(setDatasetStatus(out, 'VALID')) tier = out.split('/')[-1] to_DDM = (wl['RequestType'] == 'ReDigi' and not ('DQM' in tier)) campaign = None try: campaign = out.split('/')[2].split('-')[0] except: if 'Campaign' in wl and wl['Campaign']: campaign = wl['Campaign'] if campaign and campaign in CI.campaigns and 'toDDM' in CI.campaigns[campaign] and tier in CI.campaigns[campaign]['toDDM']: to_DDM = True ## inject to DDM when necessary passed_to_DDM=True if to_DDM: #print "Sending",out," to DDM" status = subprocess.call(['python','assignDatasetToSite.py','--nCopies=2','--dataset='+out,'--exec']) if status!=0: print "Failed DDM, retrying a second time" status = subprocess.call(['python','assignDatasetToSite.py','--nCopies=2','--dataset='+out,'--exec']) if status!=0: results.append("Failed DDM for %s"% out) sendEmail("failed DDM injection","could not add "+out+" to DDM pool. check closor logs.") passed_to_DDM=False if passed_to_DDM: ## make a lock release LI.release_everywhere( out, reason = 'global unlock after passing to DDM') pass else: print wfo.name,"no stats for announcing",out results.append('No Stats') if all(map(lambda result : result in ['None',None,True],results)): ## only announce if all previous are fine results.append(reqMgrClient.announceWorkflowCascade(url, wfo.name)) #print results if all(map(lambda result : result in ['None',None,True],results)): wfo.status = 'done' session.commit() print wfo.name,"is announced" else: print "ERROR with ",wfo.name,"to be announced",json.dumps( results ) else: print wfo.name,"not good for announcing:",wl['RequestStatus']
def closor(url, specific=None): CI = campaignInfo() ## manually closed-out workflows should get to close with checkor for wfo in session.query(Workflow).filter(Workflow.status=='close').all(): if specific and not specific in wfo.name: continue ## what is the expected #lumis wl = getWorkLoad(url, wfo.name) wfo.wm_status = wl['RequestStatus'] if wl['RequestStatus'] in ['announced','normal-archived']: ## manually announced ?? wfo.status = 'done' wfo.wm_status = wl['RequestStatus'] print wfo.name,"is done already",wfo.wm_status session.commit() if not 'TotalInputLumis' in wl: print wfo.name,"has not been assigned yet" continue expected_lumis = wl['TotalInputLumis'] ## what are the outputs outputs = wl['OutputDatasets'] ## check whether the number of lumis is as expected for each all_OK = [] #print outputs if len(outputs): print wfo.name,wl['RequestStatus'] for out in outputs: event_count,lumi_count = getDatasetEventsAndLumis(dataset=out) odb = session.query(Output).filter(Output.datasetname==out).first() if not odb: print "adding an output object",out odb = Output( datasetname = out ) odb.workflow = wfo session.add( odb ) odb.nlumis = lumi_count odb.nevents = event_count odb.workfow_id = wfo.id if odb.expectedlumis < expected_lumis: odb.expectedlumis = expected_lumis else: expected_lumis = odb.expectedlumis odb.date = time.mktime(time.gmtime()) session.commit() print "\t%60s %d/%d = %3.2f%%"%(out,lumi_count,expected_lumis,lumi_count/float(expected_lumis)*100.) #print wfo.fraction_for_closing, lumi_count, expected_lumis fraction = wfo.fraction_for_closing fraction = 0.0 all_OK.append((float(lumi_count) > float(expected_lumis*fraction))) ## only that status can let me go into announced if wl['RequestStatus'] in ['closed-out']: print wfo.name,"to be announced" results=[]#'dummy'] if not results: results.append(reqMgrClient.announceWorkflowCascade(url, wfo.name)) for (io,out) in enumerate(outputs): if all_OK[io]: results.append(setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', out, 'VALID' ,'')) tier = out.split('/')[-1] to_DDM = (wl['RequestType'] == 'ReDigi' and not ('DQM' in tier)) campaign = None try: campaign = out.split('/')[2].split('-')[0] except: if 'Campaign' in wl and wl['Campaign']: campaign = wl['Campaign'] if campaign and campaign in CI.campaigns and 'toDDM' in CI.campaigns[campaign] and tier in CI.campaigns[campaign]['toDDM']: to_DDM = True ## inject to DDM everything from ReDigi if to_DDM: print "Sending",out," to DDM" subprocess.call(['python','assignDatasetToSite.py','--dataset='+out,'--exec']) else: print wfo.name,"no stats for announcing",out results.append(None) #print results if all(map(lambda result : result in ['None',None],results)): wfo.status = 'done' session.commit() print wfo.name,"is announced" else: print "ERROR with ",wfo.name,"to be announced",json.dumps( results ) else: print wfo.name,"not good for announcing:",wl['RequestStatus']
def wfl(wf, view=False, p=False, ms=False, within=False, ongoing=False, status=False, update=False): wfn = wf.name wfs = wf.wm_status wl = None pid = None wl_pid = None pids = filter(lambda seg: seg.count('-') == 2, wf.name.split('_')) if len(pids): pids = pids[:1] pid = pids[0] if not pids: wl = getWL(wf.name) pids = getPrepIDs(wl) pid = pids[0] wl_pid = pid if 'task' in wf.name: wl_pid = 'task_' + pid text = ', '.join([ #wfn, #'<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a> '%(wfn,wfn), #'<table><tr><td>%s</td></tr></table>'%(wfn), #'<span>%s</span>'%(wfn), "%s " % wfn, '(%s) <br>' % wfs ]) text += ', '.join([ '<a href="https://%s/reqmgr2/fetch?rid=%s" target="_blank">dts</a>' % (reqmgr_url, wfn), '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts-req1</a>' % wfn, #TOFIX '<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>'%wfn, '<a href="https://%s/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>' % (reqmgr_url, wfn), '<a href="https://%s/reqmgr2/data/request?name=%s" target="_blank">req</a>' % (reqmgr_url, wfn), #'<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>'%wfn, #TOFIX '<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'%wfn, '<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>' % wfn, '<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>' % wfn, '<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>' % pid, '<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank">pv</a>' % wfn, #deprecated '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>'%wfn, '<a href="closeout.html#%s" target="_blank">clo</a>' % wfn, '<a href="statuses.html#%s" target="_blank">st</a>' % wfn, '<a href="https://%s/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>' % (reqmgr_url, wfn) ]) if within and (not view or wfs == 'completed'): wl = getWL(wfn) dataset = None if 'InputDataset' in wl: dataset = wl['InputDataset'] if 'Task1' in wl and 'InputDataset' in wl['Task1']: dataset = wl['Task1']['InputDataset'] if dataset: text += ', '.join([ '', '<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>' % dataset, '<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>' % dataset, '<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>' % dataset, '<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>' % dataset, ]) if p: cached = filter(lambda d: d['RequestName'] == wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad('cmsweb.cern.ch', wfn) text += ', (%s)' % (wl['RequestPriority']) pass if pid: if ms: mcm_s = json.loads( os.popen( 'curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure' % pid).read())[pid] text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>' % ( pid, mcm_s) else: text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>' % ( pid) text += ', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>' % ( wl_pid) if status: if wf.status.startswith('assistance'): text += ', <a href="assistance.html#%s" target="_blank">assist</a>' % wfn text += ' : %s ' % (wf.status) if view and wfs != 'acquired': text += '<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>' % ( wfn.replace('_', '/'), wfn.replace('_', '/')) if ongoing: text += '<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank"><img src="http://cms-gwmsmon.cern.ch/prodview/graphs/%s/daily" style="height:50px"></a>' % ( wfn, wfn) if ongoing: date1 = time.strftime( '%Y-%m-%d+%H:%M', time.gmtime(time.mktime(time.gmtime()) - (15 * 24 * 60 * 60))) date2 = time.strftime('%Y-%m-%d+%H:%M', time.gmtime()) text += '<a href="http://dashb-cms-job.cern.ch/dashboard/templates/web-job2/#table=Jobs&date1=%s&date2=%s&sortby=site&task=wmagent_%s">dashb</a>' % ( date1, date2, wfn) if ongoing and wfn in boost: for task in boost[wfn]: overflow = boost[wfn][task].get('ReplaceSiteWhitelist', None) if not overflow: overflow = boost[wfn][task].get('AddWhitelist', None) if overflow: text += ',boost (<a href=equalizor.json>%d</a>)' % len( overflow) #text+="<hr>" return text
from assignSession import * from utils import getWorkLoad import sys if __name__ == "__main__": spec = sys.argv[1] status = None if len(sys.argv)>2: status = sys.argv[2] for wf in session.query(Workflow).all(): if spec and spec not in wf.name: continue #if not wf.status in ['away']: continue wl = getWorkLoad('cmsweb.cern.ch', wf.name) wf.wm_status = wl['RequestStatus'] if status: if status == 'DELETE': print "removing",wf.name session.delete( wf ) else: wf.status = status elif wf.wm_status in ['assignment-approved']: wf.status = 'considered' elif wf.wm_status in ['assigned','acquired','running-closed','running-open','completed']: if not wf.status.startswith('assistance'): wf.status = 'away' else: print wf.name,"is still in",wf.status
from utils import getWorkLoad, reqmgr_url import sys url = reqmgr_url if __name__ == "__main__": spec = sys.argv[1] status = None if len(sys.argv)>2: status = sys.argv[2] for wf in session.query(Workflow).all(): if spec and spec not in wf.name: continue #if not wf.status in ['away']: continue wl = getWorkLoad(url, wf.name) wf.wm_status = wl['RequestStatus'] if status: if status == 'DELETE': print "removing",wf.name session.delete( wf ) else: wf.status = status elif wf.wm_status in ['assignment-approved']: wf.status = 'considered' elif wf.wm_status in ['assigned','acquired','running-closed','running-open','completed']: if not wf.status.startswith('assistance'): wf.status = 'away' else: print wf.name,"is still in",wf.status
def cleanor(url, specific=None): print "Deprecated" return if duplicateLock() : return delete_per_site = {} do_not_autoapprove = []#'T2_FR_CCIN2P3'] SI = siteInfo() CI = campaignInfo() LI = lockInfo() counts=0 for wfo in session.query(Workflow).filter(Workflow.status == 'done').all(): keep_a_copy = False if specific and not specific in wfo.name: continue ## what was in input wl = getWorkLoad(url, wfo.name ) if 'Campaign' in wl and wl['Campaign'] in CI.campaigns and 'clean-in' in CI.campaigns[wl['Campaign']] and CI.campaigns[wl['Campaign']]['clean-in']==False: print "Skipping cleaning on input for campaign",wl['Campaign'], "as per campaign configuration" continue dataset= 'N/A' if 'InputDataset' in wl: dataset = wl['InputDataset'] print dataset,"in input" #print json.dumps(wl, indent=2) announced_log = filter(lambda change : change["Status"] in ["closed-out","normal-archived","announced"],wl['RequestTransition']) if not announced_log: print "Cannot figure out when",wfo.name,"was finished" continue now = time.mktime(time.gmtime()) / (60*60*24.) then = announced_log[-1]['UpdateTime'] / (60.*60.*24.) if (now-then) <2: print "workflow",wfo.name, "finished",now-then,"days ago. Too fresh to clean" continue else: print "workflow",wfo.name,"has finished",now-then,"days ago." if not 'InputDataset' in wl: ## should we set status = clean ? or something even further print "passing along",wfo.name,"with no input" wfo.status = 'clean' session.commit() continue if 'MinBias' in dataset: print "Should not clean anything using",dataset,"setting status further" wfo.status = 'clean' session.commit() continue total_size = getDatasetSize( dataset ) ## in Gb #if counts> 20: break counts+=1 ## find any location it is at our_presence = getDatasetPresence(url, dataset, complete=None, group="DataOps") also_our_presence = getDatasetPresence(url, dataset, complete=None, group="") ## is there a custodial !!! custodials = findCustodialLocation(url, dataset) if not len(custodials): print dataset,"has no custodial site yet, excluding from cleaning" continue ## find out whether it is still in use using_the_same = getWorkflowByInput(url, dataset, details=True) conflict=False for other in using_the_same: if other['RequestName'] == wfo.name: continue if other['RequestType'] == 'Resubmission': continue if not other['RequestStatus'] in ['announced','normal-archived','aborted','rejected','aborted-archived','aborted-completed','rejected-archived','closed-out','None',None,'new']: print other['RequestName'],'is in status',other['RequestStatus'],'preventing from cleaning',dataset conflict=True break if 'Campaign' in other and other['Campaign'] in CI.campaigns and 'clean-in' in CI.campaigns[other['Campaign']] and CI.campaigns[other['Campaign']]['clean-in']==False: print other['RequestName'],'is in campaign',other['Campaign'] conflict = True break if conflict: continue print "other statuses:",[other['RequestStatus'] for other in using_the_same if other['RequestName'] != wfo.name] ## find all disks to_be_cleaned = filter(lambda site : site.startswith('T2') or site.endswith('Disk') ,our_presence.keys()) to_be_cleaned.extend( filter(lambda site : site.startswith('T2') or site.endswith('Disk') ,also_our_presence.keys())) print to_be_cleaned,"for",total_size,"GB" anaops_presence = getDatasetPresence(url, dataset, complete=None, group="AnalysisOps") own_by_anaops = anaops_presence.keys() print "Own by analysis ops and vetoing" print own_by_anaops ## need to black list the sites where there is a copy of analysis ops to_be_cleaned = [site for site in to_be_cleaned if not site in own_by_anaops ] ## keep one copy out there if 'Campaign' in wl and wl['Campaign'] in CI.campaigns and 'keep-one' in CI.campaigns[wl['Campaign']] and CI.campaigns[wl['Campaign']]['keep-one']==True: print "Keeping a copy of input for",wl['Campaign'] keep_a_copy = True if keep_a_copy: keep_at = None full_copies = [site for (site,(there,_)) in our_presence.items() if there and site.startswith('T1')] full_copies.extend( [site for (site,(there,_)) in also_our_presence.items() if there and site.startswith('T1')] ) if not full_copies: full_copies = [site for (site,(there,_)) in our_presence.items() if there and site.startswith('T2')] full_copies.extend( [site for (site,(there,_)) in also_our_presence.items() if there and site.startswith('T2')] ) if full_copies: keep_at = random.choice( full_copies ) if not keep_at: print "We are enable to find a place to keep a full copy of",dataset,"skipping" continue else: ## keeping that copy ! print "Keeping a full copy of",dataset,"at",keep_at,"not setting the status further" to_be_cleaned.remove( keep_at ) else: wfo.status = 'clean' ## collect delete request per site for site in to_be_cleaned : if not site in delete_per_site: delete_per_site[site] = [] if not dataset in [existing[0] for existing in delete_per_site[site]]: delete_per_site[site].append( (dataset, total_size) ) session.commit() #open('deletes.json','w').write( json.dumps(delete_per_site,indent=2) ) print json.dumps(delete_per_site, indent=2) print "\n\n ------- \n\n" ## unroll the deletion per site ## maybe find the optimum site/dataset dataset/site to limit the number of ph requests for site in delete_per_site: dataset_list = [info[0] for info in delete_per_site[site]] size_removal = sum([info[1] for info in delete_per_site[site]]) / 1024. if site in SI.disk: free = SI.disk[site] print site,"has",size_removal,"TB of potential cleanup.",free,"TB available." else: print site,"has",size_removal,"TB of potential cleanup. no info on available." print "\t",','.join(dataset_list) ## make deletion requests for site in delete_per_site: site_datasets = [info[0] for info in delete_per_site[site]] is_tape = any([v in site for v in ['MSS','Export','Buffer'] ]) #comments="Cleanup input after production. DataOps will take care of approving it." #if is_tape: # comments="Cleanup input after production." for item in site_datasets: LI.release( item, site, 'cleanup of input after production')
def wfl(wf, view=False, p=False, ms=False, within=False, ongoing=False, status=False, update=False): wfn = wf.name wfs = wf.wm_status wl = None pid = None pids = filter(lambda seg: seg.count("-") == 2, wf.name.split("_")) if len(pids): pids = pids[:1] pid = pids[0] if not pids: wl = getWL(wf.name) pids = getPrepIDs(wl) pid = pids[0] text = ", ".join( [ # wfn, '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a>' % (wfn, wfn), "(%s) <br>" % wfs, ] ) text += ", ".join( [ '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts</a>' % wfn, '<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>' % wfn, '<a href="https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>' % wfn, '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>' % wfn, '<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>' % wfn, '<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>' % wfn, '<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>' % wfn, '<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>' % pid, '<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank">pv</a>' % wfn, '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>' % wfn, '<a href="closeout.html#%s" target="_blank">clo</a>' % wfn, '<a href="statuses.html#%s" target="_blank">st</a>' % wfn, '<a href="https://cmsweb.cern.ch/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>' % wfn, ] ) if within and (not view or wfs == "completed"): wl = getWL(wfn) dataset = None if "InputDataset" in wl: dataset = wl["InputDataset"] if "Task1" in wl and "InputDataset" in wl["Task1"]: dataset = wl["Task1"]["InputDataset"] if dataset: text += ", ".join( [ "", "<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>" % dataset, "<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>" % dataset, "<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>" % dataset, "<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>" % dataset, ] ) if p: cached = filter(lambda d: d["RequestName"] == wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad("cmsweb.cern.ch", wfn) text += ", (%s)" % (wl["RequestPriority"]) pass if pid: if ms: mcm_s = json.loads( os.popen( "curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure" % pid ).read() )[pid] text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>' % ( pid, mcm_s, ) else: text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>' % (pid) text += ( ', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>' % (pid) ) if status: if wf.status.startswith("assistance"): text += ', <a href="assistance.html#%s" target="_blank">assist</a>' % wfn text += " : %s " % (wf.status) if view and wfs != "acquired": text += ( '<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>' % (wfn.replace("_", "/"), wfn.replace("_", "/")) ) if ongoing: text += ( '<a href="http://cms-gwmsmon.cern.ch/prodview/%s" target="_blank"><img src="http://cms-gwmsmon.cern.ch/prodview/graphs/%s/daily" style="height:50px"></a>' % (wfn, wfn) ) if ongoing: date1 = time.strftime("%Y-%m-%d+%H:%M", time.gmtime(time.mktime(time.gmtime()) - (15 * 24 * 60 * 60))) date2 = time.strftime("%Y-%m-%d+%H:%M", time.gmtime()) text += ( '<a href="http://dashb-cms-job.cern.ch/dashboard/templates/web-job2/#table=Jobs&date1=%s&date2=%s&sortby=site&task=wmagent_%s">dashb</a>' % (date1, date2, wfn) ) text += "<hr>" return text
from assignSession import * from utils import getWorkLoad import sys if __name__ == "__main__": spec = sys.argv[1] status = None if len(sys.argv) > 2: status = sys.argv[2] for wf in session.query(Workflow).all(): if spec and spec not in wf.name: continue #if not wf.status in ['away']: continue wl = getWorkLoad('cmsweb.cern.ch', wf.name) wf.wm_status = wl['RequestStatus'] if status: wf.status = status elif wf.wm_status in ['assignment-approved']: wf.status = 'considered' elif wf.wm_status in [ 'assigned', 'acquired', 'running-closed', 'running-open', 'completed' ]: if not wf.status.startswith('assistance'): wf.status = 'away' else: print wf.name, "is still in", wf.status elif wf.wm_status in ['closed-out']: wf.status = 'close'
if max_wf: wfs = wfs[:max_wf] random.shuffle( wfs ) all_blocks_at_sites = defaultdict(set) #done = json.loads(open('myblock_done.json').read()) done = {} print len(wfs),"to look the output of" for iw,wfn in enumerate(wfs): if type(wfn)==dict: wl = wfn wfn = wl['RequestName'] else: wl = getWorkLoad(url, wfn) print "%s/%s:"%(iw,len(wfs)),wfn if not wl: continue outs= wl['OutputDatasets'] for out in outs: blocks_at_sites = getDatasetBlockAndSite(url, out, group="") deletions = getDatasetOnGoingDeletion(url, out) if len(deletions): print "\t\tshould not subscribe with on-going deletions",out continue for site,blocks in blocks_at_sites.items(): if 'Buffer' in site or 'Export' in site or 'MSS' in site: continue all_blocks_at_sites[site].update( blocks )
def wfl(wf,view=False,p=False,ms=False,within=False,ongoing=False,status=False,update=False): wfn = wf.name wfs = wf.wm_status pid = None pids=filter(lambda seg: seg.count('-')==2, wf.name.split('_')) if len(pids): pid=pids[0] text=', '.join([ #wfn, '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a>'%(wfn,wfn), '(%s) <br>'%wfs]) text+=', '.join([ '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts</a>'%wfn, '<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>'%wfn, '<a href="https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>'%wfn, '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>'%wfn, '<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>'%wfn, '<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>'%wfn, '<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>'%wfn, '<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>'%pid, '<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank">pv</a>'%wfn, '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>'%wfn, '<a href="closeout.html#%s" target="_blank">clo</a>'%wfn, '<a href="statuses.html#%s" target="_blank">st</a>'%wfn, '<a href="https://cmsweb.cern.ch/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>'%wfn ]) if within and (not view or wfs=='completed'): cached = filter(lambda d : d['RequestName']==wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad('cmsweb.cern.ch',wfn) if 'InputDataset' in wl: dataset = wl['InputDataset'] text+=', '.join(['', '<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>'%dataset, '<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>'%dataset, '<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>'%dataset, '<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>'%dataset, ]) if p: cached = filter(lambda d : d['RequestName']==wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad('cmsweb.cern.ch',wfn) text+=', (%s)'%(wl['RequestPriority']) pass if pid: if ms: mcm_s = json.loads(os.popen('curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure'%pid).read())[pid] text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>'%(pid,mcm_s) else: text+=', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>'%(pid) text+=', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>'%(pid) if status: if wf.status.startswith('assistance'): text+=', <a href="assistance.html#%s" target="_blank">assist</a>'%wfn text+=' : %s '%(wf.status) if view and wfs!='acquired': text+='<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>'%(wfn.replace('_','/'),wfn.replace('_','/')) if ongoing: text+='<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank"><img src="http://hcc-briantest.unl.edu/prodview/graphs/%s/daily" style="height:50px"></a>'%(wfn,wfn) text+="<hr>" return text
if max_wf: wfs = wfs[:max_wf] random.shuffle(wfs) all_blocks_at_sites = defaultdict(set) #done = json.loads(open('myblock_done.json').read()) done = {} print len(wfs), "to look the output of" for iw, wfn in enumerate(wfs): if type(wfn) == dict: wl = wfn wfn = wl['RequestName'] else: wl = getWorkLoad(url, wfn) print "%s/%s:" % (iw, len(wfs)), wfn if not wl: continue outs = wl['OutputDatasets'] for out in outs: blocks_at_sites = getDatasetBlockAndSite(url, out, group="") deletions = getDatasetOnGoingDeletion(url, out) if len(deletions): print "\t\tshould not subscribe with on-going deletions", out continue for site, blocks in blocks_at_sites.items(): if 'Buffer' in site or 'Export' in site or 'MSS' in site: continue all_blocks_at_sites[site].update(blocks)
def injector(url, options, specific): use_mcm = True up = componentInfo(mcm=use_mcm, soft=['mcm']) if not up.check(): return use_mcm = up.status['mcm'] workflows = getWorkflows(url, status=options.wmstatus, user=options.user) workflows.extend( getWorkflows(url, status=options.wmstatus, user='******', rtype="ReReco") ) ## regardless of users, pick up all ReReco on the table print len(workflows), "in line" cannot_inject = set() ## browse for assignment-approved requests, browsed for ours, insert the diff for wf in workflows: if specific and not specific in wf: continue exists = session.query(Workflow).filter(Workflow.name == wf).first() if not exists: wfi = workflowInfo(url, wf) #wl = getWorkLoad(url, wf) ## check first that there isn't related here with something valid can_add = True ## first try at finding a match # print wfi.request familly = session.query(Workflow).filter( Workflow.name.contains(wfi.request['PrepID'])).all() if not familly: #req_familly = getWorkflowById( url, wl['PrepID']) #familly = [session.query(Workflow).filter(Workflow.name == member).first() for member in req_familly] pids = wfi.getPrepIDs() req_familly = [] for pid in pids: req_familly.extend(getWorkflowById(url, pid, details=True)) familly = [] print len(req_familly), "members" for req_member in req_familly: #print "member",req_member['RequestName'] owfi = workflowInfo(url, req_member['RequestName'], request=req_member) other_pids = owfi.getPrepIDs() if set(pids) == set(other_pids): ## this is a real match familly.extend( session.query(Workflow).filter( Workflow.name == req_member['RequestName']).all()) for lwfo in familly: if lwfo: ## we have it already if not lwfo.status in [ 'forget', 'trouble', 'forget-unlock', 'forget-out-unlock' ]: sendLog( 'injector', "Should not put %s because of %s %s" % (wf, lwfo.name, lwfo.status)) print "Should not put", wf, "because of", lwfo.name, lwfo.status cannot_inject.add(wf) can_add = False if not can_add: continue wfi.sendLog('injector', "considering %s" % wf) new_wf = Workflow(name=wf, status=options.setstatus, wm_status=options.wmstatus) session.add(new_wf) session.commit() time.sleep(0.5) else: #print "already have",wf pass if cannot_inject: #sendEmail('workflow duplicates','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject))) sendLog( 'injector', 'These workflow cannot be added in because of duplicates \n\n %s' % ('\n'.join(cannot_inject)), level='warning') ## passing a round of invalidation of what needs to be invalidated if use_mcm and (options.invalidate or True): invalidator(url) no_replacement = set() ## pick up replacements for wf in session.query(Workflow).filter( Workflow.status == 'trouble').all(): print wf.name if specific and not specific in wf.name: continue print wf.name wfi = workflowInfo(url, wf.name) wl = wfi.request #getWorkLoad(url, wf.name) familly = getWorkflowById(url, wl['PrepID']) true_familly = [] for member in familly: if member == wf.name: continue fwl = getWorkLoad(url, member) if options.replace: if member != options.replace: continue else: if fwl['RequestDate'] < wl['RequestDate']: continue if fwl['RequestType'] == 'Resubmission': continue if fwl['RequestStatus'] in ['None', None, 'new']: continue if fwl['RequestStatus'] in [ 'rejected', 'rejected-archived', 'aborted', 'aborted-archived' ]: continue true_familly.append(fwl) if len(true_familly) == 0: #sendLog('injector','%s had no replacement'%wf.name, level='critical') wfi.sendLog( 'injector', 'the workflow was found in trouble with no replacement') no_replacement.add(wf.name) continue else: wfi.sendLog( 'injector', 'the workflow was found in trouble and has a replacement') print wf.name, "has", len(familly), "familly members" print wf.name, "has", len(true_familly), "true familly members" ##we cannot have more than one of them !!! pick the last one if len(true_familly) > 1: #sendEmail('multiple wf','please take a look at injector for %s'%wf.name) sendLog('injector', 'Multiple wf in line, will take the last one for %s \n%s' % (wf.name, ', '.join(fwl['RequestName'] for fwl in true_familly)), level='critical') for fwl in true_familly[-1:]: member = fwl['RequestName'] new_wf = session.query(Workflow).filter( Workflow.name == member).first() if not new_wf: sendLog('injector', "putting %s as replacement of %s" % (member, wf.name)) status = 'away' if fwl['RequestStatus'] in ['assignment-approved']: status = 'considered' new_wf = Workflow(name=member, status=status, wm_status=fwl['RequestStatus']) wf.status = 'forget' session.add(new_wf) else: if new_wf.status == 'forget': continue sendLog( 'injector', "getting %s as replacement of %s" % (new_wf.name, wf.name)) wf.status = 'forget' for tr in session.query(Transfer).all(): if wf.id in tr.workflows_id: sw = copy.deepcopy(tr.workflows_id) sw.remove(wf.id) sw.append(new_wf.id) tr.workflows_id = sw print tr.phedexid, "got", new_wf.name if new_wf.status != 'away': print "\t setting it considered" new_wf.status = 'considered' if tr.phedexid < 0: ## set it back to positive tr.phedexid = -tr.phedexid session.commit() ## don't do that automatically #wf.status = 'forget' session.commit() if no_replacement: #sendEmail('workflow with no replacement','%s \n are dangling there'%( '\n'.join(no_replacement))) sendLog('injector', 'workflow with no replacement, %s \n are dangling there' % ('\n'.join(no_replacement)), level='critical')
def injector(url, options, specific): mlock = moduleLock() if mlock(): return use_mcm = True up = componentInfo(soft=['mcm','wtc','jira'] ) if not up.check(): return use_mcm = up.status['mcm'] UC = unifiedConfiguration() transform_keywords = UC.get('convert_to_stepchain') workflows = getWorkflows(url, status=options.wmstatus, user=options.user) for user in UC.get("user_rereco"): workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="ReReco")) for user in (options.user_relval.split(',') if options.user_relval else UC.get("user_relval")) : workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="TaskChain")) for user in (options.user_storeresults.split(',') if options.user_storeresults else UC.get("user_storeresults")) : workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="StoreResults")) print len(workflows),"in line" cannot_inject = set() to_convert = set() status_cache = defaultdict(str) ## browse for assignment-approved requests, browsed for ours, insert the diff for wf in workflows: if specific and not specific in wf: continue exists = session.query(Workflow).filter(Workflow.name == wf ).first() if not exists: wfi = workflowInfo(url, wf) ## check first that there isn't related here with something valid can_add = True ## first try at finding a match familly = session.query(Workflow).filter(Workflow.name.contains(wfi.request['PrepID'])).all() if not familly: pids = wfi.getPrepIDs() req_familly = [] for pid in pids: req_familly.extend( getWorkflowById( url, pid, details=True) ) familly = [] print len(req_familly),"members" for req_member in req_familly: #print "member",req_member['RequestName'] owfi = workflowInfo(url, req_member['RequestName'], request=req_member) other_pids = owfi.getPrepIDs() if set(pids) == set(other_pids): ## this is a real match familly.extend( session.query(Workflow).filter(Workflow.name == req_member['RequestName']).all() ) for lwfo in familly: if lwfo: ## we have it already if not lwfo.status in ['forget','trouble','forget-unlock','forget-out-unlock']: wfi.sendLog('injector',"Should not put %s because of %s %s"%( wf, lwfo.name,lwfo.status )) sendLog('injector',"Should not put %s because of %s %s"%( wf, lwfo.name,lwfo.status ), level='critical') print "Should not put",wf,"because of",lwfo.name,lwfo.status cannot_inject.add( wf ) can_add = False ## add a check on validity of input datasets _,prim,par,sec = wfi.getIO() for d in list(prim)+list(par)+list(sec): if not d in status_cache: status_cache[d] = getDatasetStatus(d) if status_cache[d] != 'VALID': wfi.sendLog('injector',"One of the input is not VALID. %s : %s"%( d, status_cache[d])) sendLog('injector',"One of the input of %s is not VALID. %s : %s"%( wf, d, status_cache[d]), level='critical') can_add = False #else: # ##make sure that all blocks get closed # closeAllBlocks(url, d) ## check for any file in phedex, to verify existence _,ph_files,_,_ = getDatasetFiles(url, d) if not ph_files and not ( 'StoreResults' == wfi.request.setdefault('RequestType',None) ): wfi.sendLog('injector',"One of the input has no file in phedex: %s" % d ) sendLog('injector',"One of the input has no file in phedex: %s"% d, level='critical') can_add = False ### ban some workflow that you don't like anymore #outputs = wfi.request['OutputDatasets'] if not can_add: continue ## temporary hack to transform specific taskchain into stepchains good_for_stepchain = wfi.isGoodToConvertToStepChain( keywords = transform_keywords) #good_for_stepchain = wfi.isGoodToConvertToStepChain( keywords = None) ## match keywords and technical constraints if (not options.no_convert) and good_for_stepchain and not wfi.isRelval(): to_convert.add( wf ) wfi.sendLog('injector','Transforming %s TaskChain into StepChain'%wf) sendEmail('convertion to stepchain','Transforming %s TaskChain into StepChain'%wf) wfi.sendLog('injector',"considering %s"%wf) new_wf = Workflow( name = wf , status = options.setstatus, wm_status = options.wmstatus) session.add( new_wf ) session.commit() time.sleep(0.5) else: #print "already have",wf pass if cannot_inject: #sendEmail('workflow duplicates','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject))) sendLog('injector','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject)), level='critical') for wf in to_convert: os.system('./Unified/rejector.py --clone --to_step --comments \"Transform to StepChain\" %s'% wf) ## passing a round of invalidation of what needs to be invalidated if use_mcm and (options.invalidate or True): invalidator(url) no_replacement = set() #print "getting all transfers" #all_transfers=session.query(Transfer).all() #print "go!" ## pick up replacements for wf in session.query(Workflow).filter(Workflow.status == 'trouble').all(): print wf.name if specific and not specific in wf.name: continue print wf.name wfi = workflowInfo(url, wf.name ) wl = wfi.request #getWorkLoad(url, wf.name) familly = getWorkflowById( url, wl['PrepID'] ) true_familly = [] for member in familly: if member == wf.name: continue fwl = getWorkLoad(url , member) if options.replace: if member != options.replace: continue else: if fwl['RequestDate'] < wl['RequestDate']: continue if fwl['RequestType']=='Resubmission': continue if fwl['RequestStatus'] in ['None',None,'new']: continue if fwl['RequestStatus'] in ['rejected','rejected-archived','aborted','aborted-archived']: continue true_familly.append( fwl ) if len(true_familly)==0: #sendLog('injector','%s had no replacement'%wf.name, level='critical') if wfi.isRelval(): #wfi.sendLog('injector','the workflow was found in trouble with no replacement. As a relval, there is no clean way to handle this.') wfi.sendLog('injector','the workflow was found in trouble with no replacement. As a relval, there is no clean way to handle this. Setting forget') wf.status = 'forget' session.commit() else: wfi.sendLog('injector','the workflow was found in trouble with no replacement') no_replacement.add( wf.name ) continue else: wfi.sendLog('injector','the workflow was found in trouble and has a replacement') print wf.name,"has",len(familly),"familly members" print wf.name,"has",len(true_familly),"true familly members" ##we cannot have more than one of them !!! pick the last one if len(true_familly)>1: #sendEmail('multiple wf','please take a look at injector for %s'%wf.name) sendLog('injector','Multiple wf in line, will take the last one for %s \n%s'%( wf.name, ', '.join(fwl['RequestName'] for fwl in true_familly)), level='critical') for fwl in true_familly[-1:]: member = fwl['RequestName'] new_wf = session.query(Workflow).filter(Workflow.name == member).first() if not new_wf: sendLog('injector',"putting %s as replacement of %s"%( member, wf.name)) status = 'away' if fwl['RequestStatus'] in ['assignment-approved']: status = 'considered' new_wf = Workflow( name = member, status = status, wm_status = fwl['RequestStatus']) wf.status = 'forget' session.add( new_wf ) else: if new_wf.status == 'forget': continue sendLog('injector',"getting %s as replacement of %s"%( new_wf.name, wf.name )) wf.status = 'forget' for tr in session.query(TransferImp).filter( TransferImp.workflow_id == wf.id).all(): ## get all transfer working for the old workflow existing = session.query(TransferImp).filter( TransferImp.phedexid == tr.phedexid).filter( TransferImp.workflow_id == new_wf.id).all() tr.active = False ## disable the old one if not existing: ## create the transfer object for the new dependency tri = TransferImp( phedexid = tr.phedexid, workflow = new_wf) session.add( tri ) session.commit() ## don't do that automatically #wf.status = 'forget' session.commit() if no_replacement: #sendEmail('workflow with no replacement','%s \n are dangling there'%( '\n'.join(no_replacement))) sendLog('injector','workflow with no replacement\n%s \n are dangling there'% ( '\n'.join(no_replacement)), level='critical')
def injector(url, options, specific): ## passing a round of invalidation of what needs to be invalidated if options.invalidate: invalidator(url) workflows = getWorkflows(url, status=options.wmstatus,user=options.user) existing = [wf.name for wf in session.query(Workflow).all()] ## browse for assignment-approved requests, browsed for ours, insert the diff for wf in workflows: if wf not in existing: print "putting",wf new_wf = Workflow( name = wf , status = options.setstatus, wm_status = options.wmstatus) session.add( new_wf ) session.commit() existing = [wf.name for wf in session.query(Workflow).all()] ## pick up replacements for wf in session.query(Workflow).filter(Workflow.status == 'trouble').all(): if specific and wf.name != specific: continue print wf.name wl = getWorkLoad(url, wf.name) familly = getWorkflowById( url, wl['PrepID'] ) if len(familly)==1: print wf.name,"ERROR has no replacement" continue print wf.name,"has",len(familly),"familly members" for member in familly: if member != wf.name: fwl = getWorkLoad(url , member) if options.replace: if member != options.replace: continue else: if fwl['RequestDate'] < wl['RequestDate']: continue if fwl['RequestType']=='Resubmission': continue if fwl['RequestStatus'] in ['None',None]: continue new_wf = session.query(Workflow).filter(Workflow.name == member).first() if not new_wf: print "putting",member status = 'away' if fwl['RequestStatus'] in ['assignment-approved']: status = 'considered' new_wf = Workflow( name = member, status = status, wm_status = fwl['RequestStatus']) wf.status = 'forget' session.add( new_wf ) session.commit() else: if new_wf.status == 'forget': continue print "getting",new_wf.name,"as replacement of",wf.name for tr in session.query(Transfer).all(): if wf.id in tr.workflows_id: sw = copy.deepcopy(tr.workflows_id) sw.remove( wf.id) sw.append(new_wf.id) tr.workflows_id = sw print tr.phedexid,"got",new_wf.name if new_wf.status != 'away': new_wf.status = 'staging' session.commit() ## don't do that automatically #wf.status = 'forget' session.commit()
def closor(url, specific=None): dbsapi = DbsApi(url='https://cmsweb.cern.ch/dbs/prod/global/DBSReader') for wfo in session.query(Workflow).filter(Workflow.status=='away').all(): if specific and not specific in wfo.name: continue ## what is the expected #lumis wl = getWorkLoad(url, wfo.name) if wfo.wm_status != wl['RequestStatus']: wfo.wm_status = wl['RequestStatus'] session.commit() take_out = ['failed','aborted','aborted-archived','rejected','rejected-archived'] if wl['RequestStatus'] in take_out: wfo.status = 'trouble' wfo.wm_status = wl['RequestStatus'] session.commit() continue if wl['RequestStatus'] in ['announced','normal-archived']: wfo.status = 'done' wfo.wm_status = wl['RequestStatus'] session.commit() continue if wl['RequestType'] == 'Resubmission': #session.delete( wl) #session.commit() print wfo.name,"can be taken out" wfo.status = 'forget' wfo.wm_status = wl['RequestStatus'] session.commit() continue if wl['RequestStatus'] in ['assigned','acquired']: print wfo.name,"not running yet" continue if not 'TotalInputLumis' in wl: print wfo.name,"has not been assigned yet" continue expected_lumis = wl['TotalInputLumis'] ## what are the outputs outputs = wl['OutputDatasets'] ## check whether the number of lumis is as expected for each all_OK = [] #print outputs if len(outputs): print wfo.name,wl['RequestStatus'] for out in outputs: reply = dbsapi.listFileSummaries(dataset=out) lumi_count = 0 event_count = 0 for f in reply: lumi_count +=f['num_lumi'] event_count +=f['num_event'] odb = session.query(Output).filter(Output.datasetname==out).first() if not odb: print "adding an output object",out odb = Output( datasetname = out ) odb.workflow = wfo session.add( odb ) odb.nlumis = lumi_count odb.nevents = event_count if odb.expectedlumis < expected_lumis: odb.expectedlumis = expected_lumis else: expected_lumis = odb.expectedlumis odb.date = time.mktime(time.gmtime()) session.commit() print "\t%60s %d/%d = %3.2f%%"%(out,lumi_count,expected_lumis,lumi_count/float(expected_lumis)*100.) #print wfo.fraction_for_closing, lumi_count, expected_lumis fraction = wfo.fraction_for_closing fraction = 0.05 all_OK.append((float(lumi_count) > float(expected_lumis*fraction))) ## only that status can let me go into announced if wl['RequestStatus'] in ['closed-out']: ## add force-completed ?? print wfo.name,"to be announced" results=[]#'dummy'] if not results: results.append(reqMgrClient.announceWorkflowCascade(url, wfo.name)) for (io,out) in enumerate(outputs): if all_OK[io]: results.append(setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', out, 'VALID' ,'')) ## inject to DDM everything from ReDigi if wl['RequestType'] == 'ReDigi' and not ('/DQM' in out): print "Sending",out," to DDM" subprocess.call(['python','assignDatasetToSite.py','--dataset='+out,'--exec']) else: print wfo.name,"no stats for announcing",out results.append(None) #print results if all(map(lambda result : result in ['None',None],results)): wfo.status = 'done' session.commit() print wfo.name,"is announced" else: print "ERROR with ",wfo.name,"to be announced",json.dumps( results ) else: print wfo.name,"not good for announcing:",wl['RequestStatus']
def injector(url, options, specific): use_mcm = True up = componentInfo( mcm = use_mcm, soft=['mcm'] ) if not up.check(): return use_mcm = up.status['mcm'] workflows = getWorkflows(url, status=options.wmstatus, user=options.user) workflows.extend( getWorkflows(url, status=options.wmstatus, user='******', rtype="ReReco")) ## regardless of users, pick up all ReReco on the table existing = [wf.name for wf in session.query(Workflow).all()] ## browse for assignment-approved requests, browsed for ours, insert the diff for wf in workflows: if wf not in existing: print "putting",wf new_wf = Workflow( name = wf , status = options.setstatus, wm_status = options.wmstatus) session.add( new_wf ) session.commit() time.sleep(1) existing = [wf.name for wf in session.query(Workflow).all()] ## passing a round of invalidation of what needs to be invalidated if use_mcm and (options.invalidate or True): invalidator(url) ## pick up replacements for wf in session.query(Workflow).filter(Workflow.status == 'trouble').all(): if specific and wf.name != specific: continue print wf.name wl = getWorkLoad(url, wf.name) familly = getWorkflowById( url, wl['PrepID'] ) true_familly = [] for member in familly: if member == wf.name: continue fwl = getWorkLoad(url , member) if options.replace: if member != options.replace: continue else: if fwl['RequestDate'] < wl['RequestDate']: continue if fwl['RequestType']=='Resubmission': continue if fwl['RequestStatus'] in ['None',None]: continue true_familly.append( fwl ) if len(true_familly)==0: print wf.name,"ERROR has no replacement" known = [] try: known = json.loads(open('no_replacement.json').read()) except: pass if not wf.name in known: sendEmail('workflow in %s with no replacement'%(wl['RequestStatus']),'%s is dangling there'%(wf.name)) known.append( wf.name ) open('no_replacement.json','w').write( json.dumps( known, indent=2 )) continue print wf.name,"has",len(familly),"familly members" print wf.name,"has",len(true_familly),"true familly members" for fwl in true_familly: member = fwl['RequestName'] new_wf = session.query(Workflow).filter(Workflow.name == member).first() if not new_wf: print "putting",member,"as replacement of",wf.name status = 'away' if fwl['RequestStatus'] in ['assignment-approved']: status = 'considered' new_wf = Workflow( name = member, status = status, wm_status = fwl['RequestStatus']) wf.status = 'forget' session.add( new_wf ) else: if new_wf.status == 'forget': continue print "getting",new_wf.name,"as replacement of",wf.name wf.status = 'forget' for tr in session.query(Transfer).all(): if wf.id in tr.workflows_id: sw = copy.deepcopy(tr.workflows_id) sw.remove( wf.id) sw.append(new_wf.id) tr.workflows_id = sw print tr.phedexid,"got",new_wf.name if new_wf.status != 'away': print "\t setting it considered" new_wf.status = 'considered' if tr.phedexid<0: ## set it back to positive tr.phedexid = -tr.phedexid session.commit() ## don't do that automatically #wf.status = 'forget' session.commit()
def injector(url, options, specific): mlock = moduleLock() if mlock(): return use_mcm = True up = componentInfo(soft=['mcm', 'wtc']) if not up.check(): return use_mcm = up.status['mcm'] UC = unifiedConfiguration() transform_keywords = UC.get('convert_to_stepchain') workflows = getWorkflows(url, status=options.wmstatus, user=options.user) for user in UC.get("user_rereco"): workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="ReReco")) for user in (options.user_relval.split(',') if options.user_relval else UC.get("user_relval")): workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="TaskChain")) for user in (options.user_storeresults.split(',') if options.user_storeresults else UC.get("user_storeresults")): workflows.extend( getWorkflows(url, status=options.wmstatus, user=user, rtype="StoreResults")) print len(workflows), "in line" cannot_inject = set() to_convert = set() status_cache = defaultdict(str) ## browse for assignment-approved requests, browsed for ours, insert the diff for wf in workflows: if specific and not specific in wf: continue exists = session.query(Workflow).filter(Workflow.name == wf).first() if not exists: wfi = workflowInfo(url, wf) ## check first that there isn't related here with something valid can_add = True ## first try at finding a match familly = session.query(Workflow).filter( Workflow.name.contains(wfi.request['PrepID'])).all() if not familly: pids = wfi.getPrepIDs() req_familly = [] for pid in pids: req_familly.extend(getWorkflowById(url, pid, details=True)) familly = [] print len(req_familly), "members" for req_member in req_familly: #print "member",req_member['RequestName'] owfi = workflowInfo(url, req_member['RequestName'], request=req_member) other_pids = owfi.getPrepIDs() if set(pids) == set(other_pids): ## this is a real match familly.extend( session.query(Workflow).filter( Workflow.name == req_member['RequestName']).all()) for lwfo in familly: if lwfo: ## we have it already if not lwfo.status in [ 'forget', 'trouble', 'forget-unlock', 'forget-out-unlock' ]: wfi.sendLog( 'injector', "Should not put %s because of %s %s" % (wf, lwfo.name, lwfo.status)) sendLog('injector', "Should not put %s because of %s %s" % (wf, lwfo.name, lwfo.status), level='critical') print "Should not put", wf, "because of", lwfo.name, lwfo.status cannot_inject.add(wf) can_add = False ## add a check on validity of input datasets _, prim, par, sec = wfi.getIO() for d in list(prim) + list(par) + list(sec): if not d in status_cache: status_cache[d] = getDatasetStatus(d) if status_cache[d] != 'VALID': wfi.sendLog( 'injector', "One of the input is not VALID. %s : %s" % (d, status_cache[d])) sendLog('injector', "One of the input of %s is not VALID. %s : %s" % (wf, d, status_cache[d]), level='critical') can_add = False ## check for any file in phedex, to verify existence _, ph_files, _, _ = getDatasetFiles(url, d) if not ph_files and not ('StoreResults' == wfi.request.setdefault( 'RequestType', None)): wfi.sendLog( 'injector', "One of the input has no file in phedex: %s" % d) sendLog('injector', "One of the input has no file in phedex: %s" % d, level='critical') can_add = False ### ban some workflow that you don't like anymore #outputs = wfi.request['OutputDatasets'] if not can_add: continue ## temporary hack to transform specific taskchain into stepchains #good_for_stepchain = wfi.isGoodToConvertToStepChain( keywords = transform_keywords) good_for_stepchain = wfi.isGoodToConvertToStepChain(keywords=None) ## match keywords and technical constraints #if (not options.no_convert) and good_for_stepchain and not wfi.isRelval(): # to_convert.add( wf ) # wfi.sendLog('injector','Transforming %s TaskChain into StepChain'%wf) # #sendEmail('convertion to stepchain','Transforming %s TaskChain into StepChain'%wf) wfi.sendLog('injector', "considering %s" % wf) new_wf = Workflow(name=wf, status=options.setstatus, wm_status=options.wmstatus) session.add(new_wf) session.commit() time.sleep(0.5) else: #print "already have",wf pass if cannot_inject: #sendEmail('workflow duplicates','These workflow cannot be added in because of duplicates \n\n %s'%( '\n'.join(cannot_inject))) sendLog( 'injector', 'These workflow cannot be added in because of duplicates \n\n %s' % ('\n'.join(cannot_inject)), level='warning') for wf in to_convert: os.system( './Unified/rejector.py --clone --to_step --comments \"Transform to StepChain\" %s' % wf) ## passing a round of invalidation of what needs to be invalidated if use_mcm and (options.invalidate or True): invalidator(url) no_replacement = set() #print "getting all transfers" #all_transfers=session.query(Transfer).all() #print "go!" ## pick up replacements for wf in session.query(Workflow).filter( Workflow.status == 'trouble').all(): print wf.name if specific and not specific in wf.name: continue print wf.name wfi = workflowInfo(url, wf.name) wl = wfi.request #getWorkLoad(url, wf.name) familly = getWorkflowById(url, wl['PrepID']) true_familly = [] for member in familly: if member == wf.name: continue fwl = getWorkLoad(url, member) if options.replace: if member != options.replace: continue else: if fwl['RequestDate'] < wl['RequestDate']: continue if fwl['RequestType'] == 'Resubmission': continue if fwl['RequestStatus'] in ['None', None, 'new']: continue if fwl['RequestStatus'] in [ 'rejected', 'rejected-archived', 'aborted', 'aborted-archived' ]: continue true_familly.append(fwl) if len(true_familly) == 0: #sendLog('injector','%s had no replacement'%wf.name, level='critical') if wfi.isRelval(): #wfi.sendLog('injector','the workflow was found in trouble with no replacement. As a relval, there is no clean way to handle this.') wfi.sendLog( 'injector', 'the workflow was found in trouble with no replacement. As a relval, there is no clean way to handle this. Setting forget' ) wf.status = 'forget' session.commit() else: wfi.sendLog( 'injector', 'the workflow was found in trouble with no replacement') no_replacement.add(wf.name) continue else: wfi.sendLog( 'injector', 'the workflow was found in trouble and has a replacement') print wf.name, "has", len(familly), "familly members" print wf.name, "has", len(true_familly), "true familly members" ##we cannot have more than one of them !!! pick the last one if len(true_familly) > 1: #sendEmail('multiple wf','please take a look at injector for %s'%wf.name) sendLog('injector', 'Multiple wf in line, will take the last one for %s \n%s' % (wf.name, ', '.join(fwl['RequestName'] for fwl in true_familly)), level='critical') for fwl in true_familly[-1:]: member = fwl['RequestName'] new_wf = session.query(Workflow).filter( Workflow.name == member).first() if not new_wf: sendLog('injector', "putting %s as replacement of %s" % (member, wf.name)) status = 'away' if fwl['RequestStatus'] in ['assignment-approved']: status = 'considered' new_wf = Workflow(name=member, status=status, wm_status=fwl['RequestStatus']) wf.status = 'forget' session.add(new_wf) else: if new_wf.status == 'forget': continue sendLog( 'injector', "getting %s as replacement of %s" % (new_wf.name, wf.name)) wf.status = 'forget' for tr in session.query(TransferImp).filter( TransferImp.workflow_id == wf.id).all(): ## get all transfer working for the old workflow existing = session.query(TransferImp).filter( TransferImp.phedexid == tr.phedexid).filter( TransferImp.workflow_id == new_wf.id).all() tr.active = False ## disable the old one if not existing: ## create the transfer object for the new dependency tri = TransferImp(phedexid=tr.phedexid, workflow=new_wf) session.add(tri) session.commit() ## don't do that automatically #wf.status = 'forget' session.commit() if no_replacement: #sendEmail('workflow with no replacement','%s \n are dangling there'%( '\n'.join(no_replacement))) sendLog('injector', 'workflow with no replacement\n%s \n are dangling there' % ('\n'.join(no_replacement)), level='critical')
def wfl(wf, view=False, p=False, ms=False, within=False, ongoing=False, status=False, update=False): wfn = wf.name wfs = wf.wm_status pid = None pids = filter(lambda seg: seg.count('-') == 2, wf.name.split('_')) if len(pids): pid = pids[0] text = ', '.join([ #wfn, '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">%s</a>' % (wfn, wfn), '(%s) <br>' % wfs ]) text += ', '.join([ '<a href="https://cmsweb.cern.ch/reqmgr/view/details/%s" target="_blank">dts</a>' % wfn, '<a href=https://cmsweb.cern.ch/reqmgr/view/showWorkload?requestName=%s target="_blank">wkl</a>' % wfn, '<a href="https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/%s" target="_blank">wfc</a>' % wfn, '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/request?requestName=%s" target="_blank">dwkc</a>' % wfn, '<a href="https://cmsweb.cern.ch/reqmgr/view/splitting/%s" target="_blank">spl</a>' % wfn, '<a href="https://cms-pdmv.cern.ch/stats/?RN=%s" target="_blank">vw</a>' % wfn, '<a href="https://cms-pdmv.cern.ch/stats/restapi/get_one/%s" target="_blank">vwo</a>' % wfn, '<a href="https://cms-logbook.cern.ch/elog/Workflow+processing/?mode=full&reverse=0&reverse=1&npp=20&subtext=%s&sall=q" target="_blank">elog</a>' % pid, '<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank">pv</a>' % wfn, '<a href="https://cmsweb.cern.ch/reqmgr/reqMgr/outputDatasetsByRequestName/%s" target="_blank">out</a>' % wfn, '<a href="closeout.html#%s" target="_blank">clo</a>' % wfn, '<a href="statuses.html#%s" target="_blank">st</a>' % wfn, '<a href="https://cmsweb.cern.ch/couchdb/workloadsummary/_design/WorkloadSummary/_show/histogramByWorkflow/%s" target="_blank">perf</a>' % wfn ]) if within and (not view or wfs == 'completed'): cached = filter(lambda d: d['RequestName'] == wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad('cmsweb.cern.ch', wfn) if 'InputDataset' in wl: dataset = wl['InputDataset'] text += ', '.join([ '', '<a href=https://cmsweb.cern.ch/das/request?input=%s target=_blank>input</a>' % dataset, '<a href=https://cmsweb.cern.ch/phedex/prod/Data::Subscriptions#state=create_since=0;filter=%s target=_blank>sub</a>' % dataset, '<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/subscriptions?dataset=%s&collapse=n target=_blank>ds</a>' % dataset, '<a href=https://cmsweb.cern.ch/phedex/datasvc/xml/prod/blockreplicas?dataset=%s target=_blank>rep</a>' % dataset, ]) if p: cached = filter(lambda d: d['RequestName'] == wfn, cache) if cached: wl = cached[0] else: wl = getWorkLoad('cmsweb.cern.ch', wfn) text += ', (%s)' % (wl['RequestPriority']) pass if pid: if ms: mcm_s = json.loads( os.popen( 'curl https://cms-pdmv.cern.ch/mcm/public/restapi/requests/get_status/%s --insecure' % pid).read())[pid] text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm (%s)</a>' % ( pid, mcm_s) else: text += ', <a href="https://cms-pdmv.cern.ch/mcm/requests?prepid=%s" target="_blank">mcm</a>' % ( pid) text += ', <a href="https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=%s" target="_blank">ac</a>' % ( pid) if status: if wf.status.startswith('assistance'): text += ', <a href="assistance.html#%s" target="_blank">assist</a>' % wfn text += ' : %s ' % (wf.status) if view and wfs != 'acquired': text += '<a href="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" target="_blank"><img src="https://cms-pdmv.web.cern.ch/cms-pdmv/stats/growth/%s.gif" style="height:50px"></a>' % ( wfn.replace('_', '/'), wfn.replace('_', '/')) if ongoing: text += '<a href="http://hcc-briantest.unl.edu/prodview/%s" target="_blank"><img src="http://hcc-briantest.unl.edu/prodview/graphs/%s/daily" style="height:50px"></a>' % ( wfn, wfn) text += "<hr>" return text