def rejector(url, specific, options=None): up = componentInfo(soft=['wtc', 'jira']) #if not up.check(): return if specific and specific.startswith('/'): ## this is for a dataset print setDatasetStatus(specific, 'INVALID') return if options.filelist: wfs = [] for line in filter(None, open(options.filelist).read().split('\n')): print line wfs.extend( session.query(Workflow).filter( Workflow.name.contains(line)).all()) elif specific: wfs = session.query(Workflow).filter( Workflow.name.contains(specific)).all() if not wfs: batches = batchInfo().content() for bname in batches: if specific == bname: for pid in batches[bname]: b_wfs = getWorkflowById(url, pid) for wf in b_wfs: wfs.append( session.query(Workflow).filter( Workflow.name == wf).first()) break else: wfs = session.query(Workflow).filter( Workflow.status == 'assistance-clone').all() #wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all()) ## be careful then on clone case by case options.clone = True print "not supposed to function yet" return print len(wfs), "to reject" if len(wfs) > 1: print "\n".join([wfo.name for wfo in wfs]) answer = raw_input('Reject these') if not answer.lower() in ['y', 'yes']: return for wfo in wfs: #wfo = session.query(Workflow).filter(Workflow.name == specific).first() if not wfo: print "cannot reject", spec return wfi = workflowInfo(url, wfo.name) comment = "" if options.comments: comment = ", reason: " + options.comments if options.keep: wfi.sendLog( 'rejector', 'invalidating the workflow by unified operator%s' % comment) else: wfi.sendLog( 'rejector', 'invalidating the workflow and outputs by unified operator%s' % comment) results = invalidate(url, wfi, only_resub=True, with_output=(not options.keep)) if all(results): print wfo.name, "rejected" if options and options.clone: wfo.status = 'trouble' session.commit() schema = wfi.getSchema() schema['Requestor'] = os.getenv('USER') schema['Group'] = 'DATAOPS' schema['OriginalRequestName'] = wfo.name if 'ProcessingVersion' in schema: schema['ProcessingVersion'] = int( schema['ProcessingVersion'] ) + 1 ## dubious str->int conversion else: schema['ProcessingVersion'] = 2 for k in schema.keys(): if k.startswith('Team'): schema.pop(k) if k.startswith('checkbox'): schema.pop(k) ## a few tampering of the original request if options.Memory: if schema['RequestType'] == 'TaskChain': it = 1 while True: t = 'Task%d' % it it += 1 if t in schema: schema[t]['Memory'] = options.Memory else: break else: schema['Memory'] = options.Memory if options.short_task and schema['RequestType'] == 'TaskChain': translate = {} it = 1 while True: tt = 'Task%d' % it if tt in schema: tname = schema[tt]['TaskName'] ntname = 'T%d' % it translate[tname] = ntname it += 1 schema[tt]['TaskName'] = ntname if 'InputTask' in schema[tt]: itname = schema[tt]['InputTask'] schema[tt]['InputTask'] = translate[itname] else: break for k in schema.get('ProcessingString', {}).keys(): schema['ProcessingString'][ translate[k]] = schema['ProcessingString'].pop(k) for k in schema.get('AcquisitionEra', {}).keys(): schema['AcquisitionEra'][ translate[k]] = schema['AcquisitionEra'].pop(k) if options.Multicore: ## to do : set it properly in taskchains if schema['RequestType'] == 'TaskChain': tasks, set_to = options.Multicore.split( ':') if ':' in options.Multicore else ( "", options.Multicore) set_to = int(set_to) tasks = tasks.split(',') if tasks else ['Task1'] it = 1 while True: tt = 'Task%d' % it it += 1 if tt in schema: tname = schema[tt]['TaskName'] if tname in tasks or tt in tasks: mem = schema[tt]['Memory'] mcore = schema[tt].get('Multicore', 1) factor = (set_to / float(mcore)) fraction_constant = 0.4 mem_per_core_c = int( (1 - fraction_constant) * mem / float(mcore)) print "mem per core", mem_per_core_c print "base mem", mem ## adjusting the parameter in the clone schema[tt]['Memory'] += ( set_to - mcore) * mem_per_core_c schema[tt]['Multicore'] = set_to schema[tt]['TimePerEvent'] /= factor else: break else: schema['Multicore'] = options.Multicore if options.deterministic: if schema['RequestType'] == 'TaskChain': schema['Task1']['DeterministicPileup'] = True if options.EventsPerJob: if schema['RequestType'] == 'TaskChain': schema['Task1']['EventsPerJob'] = options.EventsPerJob else: schema['EventsPerJob'] = options.EventsPerJob if options.EventAwareLumiBased: schema['SplittingAlgo'] = 'EventAwareLumiBased' if options.TimePerEvent: schema['TimePerEvent'] = options.TimePerEvent if options.ProcessingString: schema['ProcessingString'] = options.ProcessingString if options.AcquisitionEra: schema['AcquisitionEra'] = options.AcquisitionEra if options.runs: schema['RunWhitelist'] = map(int, options.runs.split(',')) if options.PrepID: schema['PrepID'] = options.PrepID if schema['RequestType'] == 'TaskChain' and options.no_output: ntask = schema['TaskChain'] for it in range(1, ntask - 1): schema['Task%d' % it]['KeepOutput'] = False schema['TaskChain'] = ntask - 1 schema.pop('Task%d' % ntask) if options.priority: schema['RequestPriority'] = options.priority ## update to the current priority schema['RequestPriority'] = wfi.request['RequestPriority'] ## drop shit on the way to reqmgr2 schema = reqMgrClient.purgeClonedSchema(schema) print "submitting" if (options.to_stepchain and (schema['RequestType'] == 'TaskChain')): ## transform the schema into StepChain schema print "Transforming a TaskChain into a StepChain" mcore = 0 mem = 0 schema['RequestType'] = 'StepChain' schema['StepChain'] = schema.pop('TaskChain') schema['SizePerEvent'] = 0 schema['TimePerEvent'] = 0 step = 1 s_n = {} while True: if 'Task%d' % step in schema: sname = 'Step%d' % step schema[sname] = schema.pop('Task%d' % step) tmcore = schema[sname].pop('Multicore') tmem = schema[sname].pop('Memory') if mcore and tmcore != mcore: wfi.sendLog( 'rejector', 'the conversion to stepchain encoutered different value of Multicore %d != %d' % (tmcore, mcore)) sendLog( 'rejector', 'the conversion of %s to stepchain encoutered different value of Multicore %d != %d' % (wfo.name, tmcore, mcore), level='critical') mcore = max(mcore, tmcore) mem = max(mem, tmem) schema[sname]['StepName'] = schema[sname].pop( 'TaskName') s_n[schema[sname]['StepName']] = sname if 'InputTask' in schema[sname]: schema[sname]['InputStep'] = schema[sname].pop( 'InputTask') eff = 1. up_s = sname while True: ## climb up a step. supposedely already all converted up_s = s_n.get( schema[up_s].get('InputStep', None), None) if up_s: ## multiply with the efficiency eff *= schema[up_s].get( 'FilterEfficiency', 1.) else: ## or stop there break if not 'KeepOutput' in schema[sname]: ## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop schema[sname]['KeepOutput'] = False schema['TimePerEvent'] += eff * schema[sname].pop( 'TimePerEvent') schema['SizePerEvent'] += eff * schema[sname].pop( 'SizePerEvent') step += 1 else: break schema['Multicore'] = mcore schema['Memory'] = mem print json.dumps(schema, indent=2) newWorkflow = reqMgrClient.submitWorkflow(url, schema) if not newWorkflow: msg = "Error in cloning {}".format(wfo.name) print(msg) wfi.sendLog('rejector', msg) # Get the error message time.sleep(5) data = reqMgrClient.requestManagerPost( url, "/reqmgr2/data/request", schema) wfi.sendLog('rejector', data) print json.dumps(schema, indent=2) return print newWorkflow data = reqMgrClient.setWorkflowApproved(url, newWorkflow) print data wfi.sendLog( 'rejector', 'Cloned into %s by unified operator %s' % (newWorkflow, comment)) #wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False) else: wfo.status = 'trouble' if options.set_trouble else 'forget' wfi.notifyRequestor('Rejected by unified operator %s' % (comment), do_batch=False) session.commit() else: msg = "Error in rejecting {}: {}".format(wfo.name, results) print(msg) wfi.sendLog('rejector', msg)
def singleClone(url, wfname, actions, comment, do=False): wfi = workflowInfo(url, wfname) payload = wfi.getSchema() initial = wfi.request payload['Requestor'] = os.getenv('USER') payload['Group'] = 'DATAOPS' payload['OriginalRequestName'] = initial['RequestName'] payload['RequestPriority'] = initial['RequestPriority'] if 'ProcessingVersion' in initial: payload['ProcessingVersion'] = int(initial['ProcessingVersion']) + 1 else: payload['ProcessingVersion'] = 2 payload = reqMgrClient.purgeClonedSchema(payload) if actions: for action in actions: if action.startswith('mem') and actions[action] != "" and actions[ action] != 'Same': if 'TaskChain' in payload: print "Setting memory for clone of task chain" mem_dict = {} it = 1 while True: t = 'Task%d' % it it += 1 if t in payload: tname = payload[t]['TaskName'] mem_dict[tname] = int(actions[action]) print "Memory set for Task%d" % it else: break payload['Memory'] = mem_dict else: print "Setting memory for non-taskchain workflow" payload['Memory'] = int(actions[action]) print "Memory set to " + actions[action] print "Clone payload" # print json.dumps( payload , indent=2) print actions #Create clone clone = reqMgrClient.submitWorkflow(url, payload) if not clone: print "Error in making clone for", initial["RequestName"] clone = reqMgrClient.submitWorkflow(url, payload) if not clone: print "Error twice in making clone for", initial["RequestName"] sendLog('actor', 'Failed to make a clone twice for %s!' % initial["RequestName"], level='critical') wfi.sendLog( 'actor', 'Failed to make a clone twice for %s!' % initial["RequestName"]) return None if actions: for action in actions: if action.startswith('split'): cloneinfo = workflowInfo(url, clone) splittings = cloneinfo.getSplittingsNew(strip=True) if actions[action] != 'Same' and actions[ action] != 'max' and actions[action] != '': factor = int( actions[action][0:-1]) if 'x' in actions[action] else 2 for split in splittings: split_par = split['splitParams'] for act in [ 'avg_events_per_job', 'events_per_job', 'lumis_per_job' ]: if act in split_par: wfi.sendLog( 'actor', 'Changing %s (%d) by a factor %d' % (act, split_par[act], factor)) split_par[act] /= factor print "to", split_par[act] break #split['requestName'] = clone #print "changing the splitting of",clone #print json.dumps( split, indent=2 ) #print reqMgrClient.setWorkflowSplitting(url, clone, split ) elif 'max' in actions[action]: for split in splittings: split_par = split['splitParams'] for act in [ 'avg_events_per_job', 'events_per_job', 'lumis_per_job' ]: if act in split_par: wfi.sendLog( 'actor', 'Max splitting set for %s (%d' % (act, split_par[act])) print "Changing %s (%d) " % (act, split_par[act]), split_par[act] = 1 print "to max splitting ", split_par[act] break #split['requestName'] = clone #print "changing the splitting of",clone #print json.dumps( split, indent=2 ) #print reqMgrClient.setWorkflowSplitting(url, clone, split ) print "changing the splitting of", clone print json.dumps(splittings, indent=2) print reqMgrClient.setWorkflowSplitting(url, clone, splittings) #Approve data = reqMgrClient.setWorkflowApproved(url, clone) #wfi.sendLog('actor','Cloned into %s'%clone) # wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment )) # wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False) print data return clone
def modifySchema(cache, workflow, user, group, events, firstLumi, backfill=False, memory=None, timeperevent=None): """ Adapts schema to right parameters. If the original workflow points to DBS2, DBS3 URL is fixed instead. if backfill is True, modifies RequestString, ProcessingString, AcquisitionEra and Campaign to say Backfill, and restarts requestDate. """ result = reqMgrClient.purgeClonedSchema(cache) ## then further drop nested arguments taskParamBlacklist = ['EventsPerJob'] for i in range(1, 100): t = 'Task%s' % i if not t in result: break for p in taskParamBlacklist: if p in result[t]: result[t].pop(p) if memory: result['Memory'] = memory if timeperevent: result['TimePerEvent'] = timeperevent if user: result['Requestor'] = user if group: result['Group'] = group # and required for cross-cloning between prod and testbed if result.get('CouchURL'): result['ConfigCacheUrl'] = result.pop('CouchURL') #if we are extending the workflow if events: # extend workflow so it will safely start outside of the boundary RequestNumEvents = int(result['RequestNumEvents']) FirstEvent = int(result['FirstEvent']) # FirstEvent_NEW > FirstEvent + RequestNumEvents # the fist event needs to be oustide the range result['FirstEvent'] = FirstEvent + RequestNumEvents + DELTA_EVENTS # FirstLumi_NEW > FirstLumi + RequestNumEvents/events_per_job/filterEff # same for the first lumi, needs to be after the last lumi result['FirstLumi'] = firstLumi + DELTA_LUMIS # only the desired events result['RequestNumEvents'] = events # prepend EXT_ to recognize as extension result["RequestString"] = 'EXT_' + result["RequestString"] else: try: #result["ProcessingVersion"] = int(helper.getProcessingVersion()) + 1 result["ProcessingVersion"] += 1 except ValueError: result["ProcessingVersion"] = 2 # modify for backfill if backfill: # Modify ProcessingString, AcquisitionEra, Campaign and Request string (if they don't # have the word 'backfill' in it result["ProcessingString"] = "BACKFILL" if isinstance(result["AcquisitionEra"], dict): for eraName in result["AcquisitionEra"]: if "backfill" not in result["AcquisitionEra"][eraName].lower(): result["AcquisitionEra"][eraName] = result[ "AcquisitionEra"][eraName] + "Backfill" else: if "backfill" not in result["AcquisitionEra"].lower(): result[ "AcquisitionEra"] = result["AcquisitionEra"] + "Backfill" if "backfill" not in result["Campaign"].lower(): result["Campaign"] = result["Campaign"] + "-Backfill" if "backfill" not in result["RequestString"].lower(): # Word backfill in the middle of the request strin parts = result["RequestString"].split('-') result["RequestString"] = '-'.join(parts[:2] + ["Backfill"] + parts[2:]) if "PrepID" in result: # delete entry del result["PrepID"] if result["RequestType"] == 'TaskChain': for taskNumber in range(1, result['TaskChain'] + 1): taskName = 'Task%s' % taskNumber if 'ProcessingString' in result[taskName]: result[taskName]['ProcessingString'] = "BACKFILL" if 'AcquisitionEra' in result[taskName]: result[taskName]['AcquisitionEra'] += "Backfill" # DataProcessing requests don't support RequestNumEvents argument anymore if 'InputDataset' in result and result['InputDataset']: result.pop('RequestNumEvents', None) # Dirty check in TaskChain and StepChains if 'Task1' in result and 'InputDataset' in result['Task1'] and result[ 'Task1']['InputDataset']: result['Task1'].pop('RequestNumEvents', None) if 'Step1' in result and 'InputDataset' in result['Step1'] and result[ 'Step1']['InputDataset']: result['Step1'].pop('RequestNumEvents', None) return result
def modifySchema(cache, workflow, user, group, events, firstLumi, backfill=False, memory=None, timeperevent=None): """ Adapts schema to right parameters. If the original workflow points to DBS2, DBS3 URL is fixed instead. if backfill is True, modifies RequestString, ProcessingString, AcquisitionEra and Campaign to say Backfill, and restarts requestDate. """ result = reqMgrClient.purgeClonedSchema( cache ) ## then further drop nested arguments taskParamBlacklist = [ 'EventsPerJob' ] for i in range(1,100): t='Task%s'%i if not t in result: break for p in taskParamBlacklist: if p in result[t]: result[t].pop( p ) if memory: result['Memory'] = memory if timeperevent: result['TimePerEvent'] = timeperevent if user: result['Requestor'] = user if group: result['Group'] = group # and required for cross-cloning between prod and testbed if result.get('CouchURL'): result['ConfigCacheUrl'] = result.pop('CouchURL') #if we are extending the workflow if events: # extend workflow so it will safely start outside of the boundary RequestNumEvents = int(result['RequestNumEvents']) FirstEvent = int(result['FirstEvent']) # FirstEvent_NEW > FirstEvent + RequestNumEvents # the fist event needs to be oustide the range result['FirstEvent'] = FirstEvent + RequestNumEvents + DELTA_EVENTS # FirstLumi_NEW > FirstLumi + RequestNumEvents/events_per_job/filterEff # same for the first lumi, needs to be after the last lumi result['FirstLumi'] = firstLumi + DELTA_LUMIS # only the desired events result['RequestNumEvents'] = events # prepend EXT_ to recognize as extension result["RequestString"] = 'EXT_' + result["RequestString"] else: try: #result["ProcessingVersion"] = int(helper.getProcessingVersion()) + 1 result["ProcessingVersion"] += 1 except ValueError: result["ProcessingVersion"] = 2 # modify for backfill if backfill: # Modify ProcessingString, AcquisitionEra, Campaign and Request string (if they don't # have the word 'backfill' in it result["ProcessingString"] = "BACKFILL" if isinstance(result["AcquisitionEra"],dict): for eraName in result["AcquisitionEra"]: if "backfill" not in result["AcquisitionEra"][eraName].lower(): result["AcquisitionEra"][eraName] = result["AcquisitionEra"][eraName] + "Backfill" else: if "backfill" not in result["AcquisitionEra"].lower(): result["AcquisitionEra"] = result["AcquisitionEra"] + "Backfill" if "backfill" not in result["Campaign"].lower(): result["Campaign"] = result["Campaign"] + "-Backfill" if "backfill" not in result["RequestString"].lower(): # Word backfill in the middle of the request strin parts = result["RequestString"].split('-') result["RequestString"] = '-'.join(parts[:2] + ["Backfill"] + parts[2:]) if "PrepID" in result: # delete entry del result["PrepID"] if result["RequestType"] == 'TaskChain': for taskNumber in range(1, result['TaskChain'] + 1): taskName = 'Task%s' % taskNumber if 'ProcessingString' in result[taskName]: result[taskName]['ProcessingString'] = "BACKFILL" if 'AcquisitionEra' in result[taskName]: result[taskName]['AcquisitionEra'] += "Backfill" # DataProcessing requests don't support RequestNumEvents argument anymore if 'InputDataset' in result and result['InputDataset']: result.pop('RequestNumEvents', None) # Dirty check in TaskChain and StepChains if 'Task1' in result and 'InputDataset' in result['Task1'] and result['Task1']['InputDataset']: result['Task1'].pop('RequestNumEvents', None) if 'Step1' in result and 'InputDataset' in result['Step1'] and result['Step1']['InputDataset']: result['Step1'].pop('RequestNumEvents', None) return result
def rejector(url, specific, options=None): up = componentInfo(soft=['wtc','jira']) if not up.check(): return if specific and specific.startswith('/'): ## this is for a dataset print setDatasetStatus(specific, 'INVALID') return if options.filelist: wfs = [] for line in filter(None, open(options.filelist).read().split('\n')): print line wfs.extend( session.query(Workflow).filter(Workflow.name.contains(line)).all()) elif specific: wfs = session.query(Workflow).filter(Workflow.name.contains(specific)).all() if not wfs: batches = batchInfo().content() for bname in batches: if specific == bname: for pid in batches[bname]: b_wfs = getWorkflowById(url, pid) for wf in b_wfs: wfs.append( session.query(Workflow).filter(Workflow.name == wf).first()) break else: wfs = session.query(Workflow).filter(Workflow.status == 'assistance-clone').all() #wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all()) ## be careful then on clone case by case options.clone = True print "not supposed to function yet" return print len(wfs),"to reject" if len(wfs)>1: print "\n".join( [wfo.name for wfo in wfs] ) answer = raw_input('Reject these') if not answer.lower() in ['y','yes']: return for wfo in wfs: #wfo = session.query(Workflow).filter(Workflow.name == specific).first() if not wfo: print "cannot reject",spec return wfi = workflowInfo(url, wfo.name) comment="" if options.comments: comment = ", reason: "+options.comments if options.keep: wfi.sendLog('rejector','invalidating the workflow by unified operator%s'%comment) else: wfi.sendLog('rejector','invalidating the workflow and outputs by unified operator%s'%comment) results = invalidate(url, wfi, only_resub=True, with_output= (not options.keep)) if all(results): print wfo.name,"rejected" if options and options.clone: wfo.status = 'trouble' session.commit() schema = wfi.getSchema() schema['Requestor'] = os.getenv('USER') schema['Group'] = 'DATAOPS' schema['OriginalRequestName'] = wfo.name if 'ProcessingVersion' in schema: schema['ProcessingVersion'] = int(schema['ProcessingVersion'])+1 ## dubious str->int conversion else: schema['ProcessingVersion']=2 for k in schema.keys(): if k.startswith('Team'): schema.pop(k) if k.startswith('checkbox'): schema.pop(k) ## a few tampering of the original request if options.Memory: if schema['RequestType'] == 'TaskChain': it=1 while True: t = 'Task%d'%it it+=1 if t in schema: schema[t]['Memory'] = options.Memory else: break else: schema['Memory'] = options.Memory if options.Multicore: ## to do : set it properly in taskchains if schema['RequestType'] == 'TaskChain': tasks,set_to = options.Multicore.split(':') if ':' in options.Multicore else ("",options.Multicore) set_to = int(set_to) tasks = tasks.split(',') if tasks else ['Task1'] it = 1 while True: tt = 'Task%d'% it it+=1 if tt in schema: tname = schema[tt]['TaskName'] if tname in tasks or tt in tasks: mem = schema[tt]['Memory'] mcore = schema[tt].get('Multicore',1) factor = (set_to / float(mcore)) fraction_constant = 0.4 mem_per_core_c = int((1-fraction_constant) * mem / float(mcore)) print "mem per core", mem_per_core_c print "base mem", mem ## adjusting the parameter in the clone schema[tt]['Memory'] += (set_to-mcore)*mem_per_core_c schema[tt]['Multicore'] = set_to schema[tt]['TimePerEvent'] /= factor else: break else: schema['Multicore'] = options.Multicore if options.deterministic: if schema['RequestType'] == 'TaskChain': schema['Task1']['DeterministicPileup'] = True if options.EventsPerJob: if schema['RequestType'] == 'TaskChain': schema['Task1']['EventsPerJob'] = options.EventsPerJob else: schema['EventsPerJob'] = options.EventsPerJob if options.EventAwareLumiBased: schema['SplittingAlgo'] = 'EventAwareLumiBased' if options.TimePerEvent: schema['TimePerEvent'] = options.TimePerEvent if options.ProcessingString: schema['ProcessingString'] = options.ProcessingString if options.AcquisitionEra: schema['AcquisitionEra'] = options.AcquisitionEra if options.runs: schema['RunWhitelist'] = map(int,options.runs.split(',')) if options.PrepID: schema['PrepID'] =options.PrepID if schema['RequestType'] == 'TaskChain' and options.no_output: ntask = schema['TaskChain'] for it in range(1,ntask-1): schema['Task%d'%it]['KeepOutput'] = False schema['TaskChain'] = ntask-1 schema.pop('Task%d'%ntask) if options.priority: schema['RequestPriority'] = options.priority ## update to the current priority schema['RequestPriority'] = wfi.request['RequestPriority'] ## drop shit on the way to reqmgr2 schema = reqMgrClient.purgeClonedSchema( schema ) print "submitting" if (options.to_stepchain and (schema['RequestType']=='TaskChain')): ## transform the schema into StepChain schema print "Transforming a TaskChain into a StepChain" mcore = 0 mem = 0 schema['RequestType'] = 'StepChain' schema['StepChain'] = schema.pop('TaskChain') schema['SizePerEvent'] = 0 schema['TimePerEvent'] = 0 step=1 s_n = {} while True: if 'Task%d'%step in schema: sname = 'Step%d'%step schema[sname] = schema.pop('Task%d'%step) tmcore = schema[sname].pop('Multicore') tmem = schema[sname].pop('Memory') if mcore and tmcore != mcore: wfi.sendLog('rejector','the conversion to stepchain encoutered different value of Multicore %d != %d'%( tmcore, mcore)) sendLog('rejector','the conversion of %s to stepchain encoutered different value of Multicore %d != %d'%( wfo.name, tmcore, mcore), level='critical') mcore = max(mcore, tmcore) mem = max(mem, tmem) schema[sname]['StepName'] = schema[sname].pop('TaskName') s_n[ schema[sname]['StepName'] ] = sname if 'InputTask' in schema[sname]: schema[sname]['InputStep'] = schema[sname].pop('InputTask') eff = 1. up_s = sname while True: ## climb up a step. supposedely already all converted up_s = s_n.get(schema[up_s].get('InputStep',None),None) if up_s: ## multiply with the efficiency eff *= schema[up_s].get('FilterEfficiency',1.) else: ## or stop there break if not 'KeepOutput' in schema[sname]: ## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop schema[sname]['KeepOutput'] = False schema['TimePerEvent'] += eff*schema[sname].pop('TimePerEvent') schema['SizePerEvent'] += eff*schema[sname].pop('SizePerEvent') step+=1 else: break schema['Multicore'] = mcore schema['Memory'] = mem print json.dumps( schema, indent=2 ) newWorkflow = reqMgrClient.submitWorkflow(url, schema) if not newWorkflow: msg = "Error in cloning {}".format(wfo.name) print(msg) wfi.sendLog('rejector',msg) # Get the error message time.sleep(5) data = reqMgrClient.requestManagerPost(url, "/reqmgr2/data/request", schema) wfi.sendLog('rejector',data) print json.dumps( schema, indent=2 ) return print newWorkflow data = reqMgrClient.setWorkflowApproved(url, newWorkflow) print data wfi.sendLog('rejector','Cloned into %s by unified operator %s'%( newWorkflow, comment )) wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False) else: wfo.status = 'trouble' if options.set_trouble else 'forget' wfi.notifyRequestor('Rejected by unified operator %s'%( comment ),do_batch=False) session.commit() else: msg = "Error in rejecting {}: {}".format(wfo.name,results) print(msg) wfi.sendLog('rejector',msg)
def rejector(url, specific, options=None): up = componentInfo(soft=['wtc']) if not up.check(): return if specific and specific.startswith('/'): ## this is for a dataset print setDatasetStatus(specific, 'INVALID') return if options.filelist: wfs = [] for line in filter(None, open(options.filelist).read().split('\n')): print line wfs.extend( session.query(Workflow).filter( Workflow.name.contains(line)).all()) elif specific: wfs = session.query(Workflow).filter( Workflow.name.contains(specific)).all() if not wfs: batches = json.loads(open('batches.json').read()) for bname in batches: if specific == bname: for wf in batches[bname]: wfs.append( session.query(Workflow).filter( Workflow.name == wf).first()) else: wfs = session.query(Workflow).filter( Workflow.status == 'assistance-clone').all() #wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all()) ## be careful then on clone case by case options.clone = True print "not supposed to function yet" return print len(wfs), "to reject" if len(wfs) > 1: print "\n".join([wfo.name for wfo in wfs]) answer = raw_input('Reject these') if not answer.lower() in ['y', 'yes']: return for wfo in wfs: #wfo = session.query(Workflow).filter(Workflow.name == specific).first() if not wfo: print "cannot reject", spec return results = [] wfi = workflowInfo(url, wfo.name) datasets = set(wfi.request['OutputDatasets']) reqMgrClient.invalidateWorkflow( url, wfo.name, current_status=wfi.request['RequestStatus']) comment = "" if options.comments: comment = ", reason: " + options.comments wfi.sendLog( 'rejector', 'invalidating the workflow by unified operator%s' % comment) ## need to find the whole familly and reject the whole gang familly = getWorkflowById(url, wfi.request['PrepID'], details=True) for fwl in familly: if fwl['RequestDate'] < wfi.request['RequestDate']: continue if fwl['RequestType'] != 'Resubmission': continue ## does not work on second order acd #if 'OriginalRequestName' in fwl and fwl['OriginalRequestName'] != wfi.request['RequestName']: continue print "rejecting", fwl['RequestName'] reqMgrClient.invalidateWorkflow( url, fwl['RequestName'], current_status=fwl['RequestStatus'], cascade=False) datasets.update(fwl['OutputDatasets']) for dataset in datasets: if options.keep: print "keeping", dataset, "in its current status" else: results.append(setDatasetStatus(dataset, 'INVALID')) pass if all(map(lambda result: result in ['None', None, True], results)): print wfo.name, "and", datasets, "are rejected" if options and options.clone: wfo.status = 'trouble' session.commit() schema = wfi.getSchema() schema['Requestor'] = os.getenv('USER') schema['Group'] = 'DATAOPS' schema['OriginalRequestName'] = wfo.name if 'ProcessingVersion' in schema: schema['ProcessingVersion'] = int( schema['ProcessingVersion'] ) + 1 ## dubious str->int conversion else: schema['ProcessingVersion'] = 2 for k in schema.keys(): if k.startswith('Team'): schema.pop(k) if k.startswith('checkbox'): schema.pop(k) ## a few tampering of the original request if options.Memory: if schema['RequestType'] == 'TaskChain': it = 1 while True: t = 'Task%d' % it it += 1 if t in schema: schema[t]['Memory'] = options.Memory else: break else: schema['Memory'] = options.Memory if options.Multicore: ## to do : set it properly in taskchains if schema['RequestType'] == 'TaskChain': tasks, set_to = options.Multicore.split( ':') if ':' in options.Multicore else ( "", options.Multicore) set_to = int(set_to) tasks = tasks.split(',') if tasks else ['Task1'] it = 1 while True: tt = 'Task%d' % it it += 1 if tt in schema: tname = schema[tt]['TaskName'] if tname in tasks or tt in tasks: mem = schema[tt]['Memory'] mcore = schema[tt].get('Multicore', 1) factor = (set_to / float(mcore)) fraction_constant = 0.4 mem_per_core_c = int( (1 - fraction_constant) * mem / float(mcore)) print "mem per core", mem_per_core_c print "base mem", mem ## adjusting the parameter in the clone schema[tt]['Memory'] += ( set_to - mcore) * mem_per_core_c schema[tt]['Multicore'] = set_to schema[tt]['TimePerEvent'] /= factor else: break else: schema['Multicore'] = options.Multicore if options.deterministic: if schema['RequestType'] == 'TaskChain': schema['Task1']['DeterministicPileup'] = True if options.EventsPerJob: if schema['RequestType'] == 'TaskChain': schema['Task1']['EventsPerJob'] = options.EventsPerJob else: schema['EventsPerJob'] = options.EventsPerJob if options.EventAwareLumiBased: schema['SplittingAlgo'] = 'EventAwareLumiBased' if options.TimePerEvent: schema['TimePerEvent'] = options.TimePerEvent if options.ProcessingString: schema['ProcessingString'] = options.ProcessingString if options.AcquisitionEra: schema['AcquisitionEra'] = options.AcquisitionEra if options.runs: schema['RunWhitelist'] = map(int, options.runs.split(',')) if options.PrepID: schema['PrepID'] = options.PrepID if schema['RequestType'] == 'TaskChain' and options.no_output: ntask = schema['TaskChain'] for it in range(1, ntask - 1): schema['Task%d' % it]['KeepOutput'] = False schema['TaskChain'] = ntask - 1 schema.pop('Task%d' % ntask) if options.priority: schema['RequestPriority'] = options.priority ## update to the current priority schema['RequestPriority'] = wfi.request['RequestPriority'] ## drop shit on the way to reqmgr2 schema = reqMgrClient.purgeClonedSchema(schema) print "submitting" if (options.to_stepchain and (schema['RequestType'] == 'TaskChain')): ## transform the schema into StepChain schema print "Transforming a TaskChain into a StepChain" schema['RequestType'] = 'StepChain' schema['StepChain'] = schema.pop('TaskChain') schema['SizePerEvent'] = 0 schema['TimePerEvent'] = 0 step = 1 s_n = {} while True: if 'Task%d' % step in schema: schema['Step%d' % step] = schema.pop('Task%d' % step) schema['Step%d' % step]['StepName'] = schema[ 'Step%d' % step].pop('TaskName') s_n[schema['Step%d' % step]['StepName']] = 'Step%d' % step if 'InputTask' in schema['Step%d' % step]: schema['Step%d' % step]['InputStep'] = schema[ 'Step%d' % step].pop('InputTask') eff = 1. up_s = 'Step%d' % step while True: ## climb up a step. supposedely already all converted up_s = s_n.get( schema[up_s].get('InputStep', None), None) if up_s: ## multiply with the efficiency eff *= schema[up_s].get( 'FilterEfficiency', 1.) else: ## or stop there break if not 'KeepOutput' in schema['Step%d' % step]: ## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop schema['Step%d' % step]['KeepOutput'] = False schema['TimePerEvent'] += eff * schema[ 'Step%d' % step].pop('TimePerEvent') schema['SizePerEvent'] += eff * schema[ 'Step%d' % step].pop('SizePerEvent') step += 1 else: break print json.dumps(schema, indent=2) newWorkflow = reqMgrClient.submitWorkflow(url, schema) if not newWorkflow: print "error in cloning", wfo.name print json.dumps(schema, indent=2) return print newWorkflow data = reqMgrClient.setWorkflowApproved(url, newWorkflow) print data wfi.sendLog( 'rejector', 'Cloned into %s by unified operator %s' % (newWorkflow, comment)) wfi.notifyRequestor('Cloned into %s by unified operator %s' % (newWorkflow, comment), do_batch=False) else: wfo.status = 'trouble' if options.set_trouble else 'forget' wfi.notifyRequestor('Rejected by unified operator %s' % (comment), do_batch=False) session.commit() else: print "error in rejecting", wfo.name, results
def singleClone(url, wfname, actions, comment, do=False): wfi = workflowInfo(url, wfname) payload = wfi.getSchema() initial = wfi.request payload['Requestor'] = os.getenv('USER') payload['Group'] = 'DATAOPS' payload['OriginalRequestName'] = initial['RequestName'] payload['RequestPriority'] = initial['RequestPriority'] if 'ProcessingVersion' in initial: payload['ProcessingVersion'] = int(initial['ProcessingVersion']) +1 else: payload['ProcessingVersion'] = 2 payload = reqMgrClient.purgeClonedSchema( payload ) if actions: for action in actions: if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same': if 'TaskChain' in payload: print "Setting memory for clone of task chain" mem_dict = {} it=1 while True: t = 'Task%d'%it it+=1 if t in payload: tname = payload[t]['TaskName'] mem_dict[tname] = int(actions[action]) print "Memory set for Task%d"%it else: break payload['Memory'] = mem_dict else: print "Setting memory for non-taskchain workflow" payload['Memory'] = int(actions[action]) print "Memory set to " + actions[action] print "Clone payload" # print json.dumps( payload , indent=2) print actions #Create clone clone = reqMgrClient.submitWorkflow(url, payload) if not clone: print "Error in making clone for",initial["RequestName"] clone = reqMgrClient.submitWorkflow(url, payload) if not clone: print "Error twice in making clone for",initial["RequestName"] sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"],level='critical') wfi.sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"]) return None if actions: for action in actions: if action.startswith('split'): cloneinfo = workflowInfo(url, clone) splittings = cloneinfo.getSplittingsNew(strip=True) if actions[action] != 'Same' and actions[action] != 'max' and actions[action] != '': factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2 for split in splittings: split_par = split['splitParams'] for act in ['avg_events_per_job','events_per_job','lumis_per_job']: if act in split_par: wfi.sendLog('actor','Changing %s (%d) by a factor %d'%( act, split_par[act], factor)) split_par[act] /= factor print "to",split_par[act] break #split['requestName'] = clone #print "changing the splitting of",clone #print json.dumps( split, indent=2 ) #print reqMgrClient.setWorkflowSplitting(url, clone, split ) elif 'max' in actions[action]: for split in splittings: split_par = split['splitParams'] for act in ['avg_events_per_job','events_per_job','lumis_per_job']: if act in split_par: wfi.sendLog('actor','Max splitting set for %s (%d'%( act, split_par[act])) print "Changing %s (%d) "%( act, split_par[act]), split_par[act] = 1 print "to max splitting ",split_par[act] break #split['requestName'] = clone #print "changing the splitting of",clone #print json.dumps( split, indent=2 ) #print reqMgrClient.setWorkflowSplitting(url, clone, split ) print "changing the splitting of",clone print json.dumps( splittings, indent=2 ) print reqMgrClient.setWorkflowSplitting(url, clone, splittings ) #Approve data = reqMgrClient.setWorkflowApproved(url, clone) #wfi.sendLog('actor','Cloned into %s'%clone) # wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment )) # wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False) print data return clone