def rejector(url, specific, options=None):
up = componentInfo(soft=['wtc', 'jira'])
#if not up.check(): return
if specific and specific.startswith('/'):
## this is for a dataset
print setDatasetStatus(specific, 'INVALID')
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend(
session.query(Workflow).filter(
Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(
Workflow.name.contains(specific)).all()
if not wfs:
batches = batchInfo().content()
for bname in batches:
if specific == bname:
for pid in batches[bname]:
b_wfs = getWorkflowById(url, pid)
for wf in b_wfs:
wfs.append(
session.query(Workflow).filter(
Workflow.name == wf).first())
break
else:
wfs = session.query(Workflow).filter(
Workflow.status == 'assistance-clone').all()
#wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
## be careful then on clone case by case
options.clone = True
print "not supposed to function yet"
return
print len(wfs), "to reject"
if len(wfs) > 1:
print "\n".join([wfo.name for wfo in wfs])
answer = raw_input('Reject these')
if not answer.lower() in ['y', 'yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject", spec
return
wfi = workflowInfo(url, wfo.name)
comment = ""
if options.comments: comment = ", reason: " + options.comments
if options.keep:
wfi.sendLog(
'rejector',
'invalidating the workflow by unified operator%s' % comment)
else:
wfi.sendLog(
'rejector',
'invalidating the workflow and outputs by unified operator%s' %
comment)
results = invalidate(url,
wfi,
only_resub=True,
with_output=(not options.keep))
if all(results):
print wfo.name, "rejected"
if options and options.clone:
wfo.status = 'trouble'
session.commit()
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(
schema['ProcessingVersion']
) + 1 ## dubious str->int conversion
else:
schema['ProcessingVersion'] = 2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
if schema['RequestType'] == 'TaskChain':
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in schema:
schema[t]['Memory'] = options.Memory
else:
break
else:
schema['Memory'] = options.Memory
if options.short_task and schema['RequestType'] == 'TaskChain':
translate = {}
it = 1
while True:
tt = 'Task%d' % it
if tt in schema:
tname = schema[tt]['TaskName']
ntname = 'T%d' % it
translate[tname] = ntname
it += 1
schema[tt]['TaskName'] = ntname
if 'InputTask' in schema[tt]:
itname = schema[tt]['InputTask']
schema[tt]['InputTask'] = translate[itname]
else:
break
for k in schema.get('ProcessingString', {}).keys():
schema['ProcessingString'][
translate[k]] = schema['ProcessingString'].pop(k)
for k in schema.get('AcquisitionEra', {}).keys():
schema['AcquisitionEra'][
translate[k]] = schema['AcquisitionEra'].pop(k)
if options.Multicore:
## to do : set it properly in taskchains
if schema['RequestType'] == 'TaskChain':
tasks, set_to = options.Multicore.split(
':') if ':' in options.Multicore else (
"", options.Multicore)
set_to = int(set_to)
tasks = tasks.split(',') if tasks else ['Task1']
it = 1
while True:
tt = 'Task%d' % it
it += 1
if tt in schema:
tname = schema[tt]['TaskName']
if tname in tasks or tt in tasks:
mem = schema[tt]['Memory']
mcore = schema[tt].get('Multicore', 1)
factor = (set_to / float(mcore))
fraction_constant = 0.4
mem_per_core_c = int(
(1 - fraction_constant) * mem /
float(mcore))
print "mem per core", mem_per_core_c
print "base mem", mem
## adjusting the parameter in the clone
schema[tt]['Memory'] += (
set_to - mcore) * mem_per_core_c
schema[tt]['Multicore'] = set_to
schema[tt]['TimePerEvent'] /= factor
else:
break
else:
schema['Multicore'] = options.Multicore
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int, options.runs.split(','))
if options.PrepID:
schema['PrepID'] = options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1, ntask - 1):
schema['Task%d' % it]['KeepOutput'] = False
schema['TaskChain'] = ntask - 1
schema.pop('Task%d' % ntask)
if options.priority:
schema['RequestPriority'] = options.priority
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
schema = reqMgrClient.purgeClonedSchema(schema)
print "submitting"
if (options.to_stepchain
and (schema['RequestType'] == 'TaskChain')):
## transform the schema into StepChain schema
print "Transforming a TaskChain into a StepChain"
mcore = 0
mem = 0
schema['RequestType'] = 'StepChain'
schema['StepChain'] = schema.pop('TaskChain')
schema['SizePerEvent'] = 0
schema['TimePerEvent'] = 0
step = 1
s_n = {}
while True:
if 'Task%d' % step in schema:
sname = 'Step%d' % step
schema[sname] = schema.pop('Task%d' % step)
tmcore = schema[sname].pop('Multicore')
tmem = schema[sname].pop('Memory')
if mcore and tmcore != mcore:
wfi.sendLog(
'rejector',
'the conversion to stepchain encoutered different value of Multicore %d != %d'
% (tmcore, mcore))
sendLog(
'rejector',
'the conversion of %s to stepchain encoutered different value of Multicore %d != %d'
% (wfo.name, tmcore, mcore),
level='critical')
mcore = max(mcore, tmcore)
mem = max(mem, tmem)
schema[sname]['StepName'] = schema[sname].pop(
'TaskName')
s_n[schema[sname]['StepName']] = sname
if 'InputTask' in schema[sname]:
schema[sname]['InputStep'] = schema[sname].pop(
'InputTask')
eff = 1.
up_s = sname
while True:
## climb up a step. supposedely already all converted
up_s = s_n.get(
schema[up_s].get('InputStep', None), None)
if up_s:
## multiply with the efficiency
eff *= schema[up_s].get(
'FilterEfficiency', 1.)
else:
## or stop there
break
if not 'KeepOutput' in schema[sname]:
## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
schema[sname]['KeepOutput'] = False
schema['TimePerEvent'] += eff * schema[sname].pop(
'TimePerEvent')
schema['SizePerEvent'] += eff * schema[sname].pop(
'SizePerEvent')
step += 1
else:
break
schema['Multicore'] = mcore
schema['Memory'] = mem
print json.dumps(schema, indent=2)
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
msg = "Error in cloning {}".format(wfo.name)
print(msg)
wfi.sendLog('rejector', msg)
# Get the error message
time.sleep(5)
data = reqMgrClient.requestManagerPost(
url, "/reqmgr2/data/request", schema)
wfi.sendLog('rejector', data)
print json.dumps(schema, indent=2)
return
print newWorkflow
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfi.sendLog(
'rejector', 'Cloned into %s by unified operator %s' %
(newWorkflow, comment))
#wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False)
else:
wfo.status = 'trouble' if options.set_trouble else 'forget'
wfi.notifyRequestor('Rejected by unified operator %s' %
(comment),
do_batch=False)
session.commit()
else:
msg = "Error in rejecting {}: {}".format(wfo.name, results)
print(msg)
wfi.sendLog('rejector', msg)
def singleClone(url, wfname, actions, comment, do=False):
wfi = workflowInfo(url, wfname)
payload = wfi.getSchema()
initial = wfi.request
payload['Requestor'] = os.getenv('USER')
payload['Group'] = 'DATAOPS'
payload['OriginalRequestName'] = initial['RequestName']
payload['RequestPriority'] = initial['RequestPriority']
if 'ProcessingVersion' in initial:
payload['ProcessingVersion'] = int(initial['ProcessingVersion']) + 1
else:
payload['ProcessingVersion'] = 2
payload = reqMgrClient.purgeClonedSchema(payload)
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[
action] != 'Same':
if 'TaskChain' in payload:
print "Setting memory for clone of task chain"
mem_dict = {}
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in payload:
tname = payload[t]['TaskName']
mem_dict[tname] = int(actions[action])
print "Memory set for Task%d" % it
else:
break
payload['Memory'] = mem_dict
else:
print "Setting memory for non-taskchain workflow"
payload['Memory'] = int(actions[action])
print "Memory set to " + actions[action]
print "Clone payload"
# print json.dumps( payload , indent=2)
print actions
#Create clone
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error in making clone for", initial["RequestName"]
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error twice in making clone for", initial["RequestName"]
sendLog('actor',
'Failed to make a clone twice for %s!' %
initial["RequestName"],
level='critical')
wfi.sendLog(
'actor', 'Failed to make a clone twice for %s!' %
initial["RequestName"])
return None
if actions:
for action in actions:
if action.startswith('split'):
cloneinfo = workflowInfo(url, clone)
splittings = cloneinfo.getSplittingsNew(strip=True)
if actions[action] != 'Same' and actions[
action] != 'max' and actions[action] != '':
factor = int(
actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
split_par = split['splitParams']
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split_par:
wfi.sendLog(
'actor',
'Changing %s (%d) by a factor %d' %
(act, split_par[act], factor))
split_par[act] /= factor
print "to", split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
elif 'max' in actions[action]:
for split in splittings:
split_par = split['splitParams']
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split_par:
wfi.sendLog(
'actor', 'Max splitting set for %s (%d' %
(act, split_par[act]))
print "Changing %s (%d) " % (act,
split_par[act]),
split_par[act] = 1
print "to max splitting ", split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
print "changing the splitting of", clone
print json.dumps(splittings, indent=2)
print reqMgrClient.setWorkflowSplitting(url, clone, splittings)
#Approve
data = reqMgrClient.setWorkflowApproved(url, clone)
#wfi.sendLog('actor','Cloned into %s'%clone)
# wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
# wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)
print data
return clone
def modifySchema(cache,
workflow,
user,
group,
events,
firstLumi,
backfill=False,
memory=None,
timeperevent=None):
"""
Adapts schema to right parameters.
If the original workflow points to DBS2, DBS3 URL is fixed instead.
if backfill is True, modifies RequestString, ProcessingString, AcquisitionEra
and Campaign to say Backfill, and restarts requestDate.
"""
result = reqMgrClient.purgeClonedSchema(cache)
## then further drop nested arguments
taskParamBlacklist = ['EventsPerJob']
for i in range(1, 100):
t = 'Task%s' % i
if not t in result: break
for p in taskParamBlacklist:
if p in result[t]:
result[t].pop(p)
if memory:
result['Memory'] = memory
if timeperevent:
result['TimePerEvent'] = timeperevent
if user:
result['Requestor'] = user
if group:
result['Group'] = group
# and required for cross-cloning between prod and testbed
if result.get('CouchURL'):
result['ConfigCacheUrl'] = result.pop('CouchURL')
#if we are extending the workflow
if events:
# extend workflow so it will safely start outside of the boundary
RequestNumEvents = int(result['RequestNumEvents'])
FirstEvent = int(result['FirstEvent'])
# FirstEvent_NEW > FirstEvent + RequestNumEvents
# the fist event needs to be oustide the range
result['FirstEvent'] = FirstEvent + RequestNumEvents + DELTA_EVENTS
# FirstLumi_NEW > FirstLumi + RequestNumEvents/events_per_job/filterEff
# same for the first lumi, needs to be after the last lumi
result['FirstLumi'] = firstLumi + DELTA_LUMIS
# only the desired events
result['RequestNumEvents'] = events
# prepend EXT_ to recognize as extension
result["RequestString"] = 'EXT_' + result["RequestString"]
else:
try:
#result["ProcessingVersion"] = int(helper.getProcessingVersion()) + 1
result["ProcessingVersion"] += 1
except ValueError:
result["ProcessingVersion"] = 2
# modify for backfill
if backfill:
# Modify ProcessingString, AcquisitionEra, Campaign and Request string (if they don't
# have the word 'backfill' in it
result["ProcessingString"] = "BACKFILL"
if isinstance(result["AcquisitionEra"], dict):
for eraName in result["AcquisitionEra"]:
if "backfill" not in result["AcquisitionEra"][eraName].lower():
result["AcquisitionEra"][eraName] = result[
"AcquisitionEra"][eraName] + "Backfill"
else:
if "backfill" not in result["AcquisitionEra"].lower():
result[
"AcquisitionEra"] = result["AcquisitionEra"] + "Backfill"
if "backfill" not in result["Campaign"].lower():
result["Campaign"] = result["Campaign"] + "-Backfill"
if "backfill" not in result["RequestString"].lower():
# Word backfill in the middle of the request strin
parts = result["RequestString"].split('-')
result["RequestString"] = '-'.join(parts[:2] + ["Backfill"] +
parts[2:])
if "PrepID" in result:
# delete entry
del result["PrepID"]
if result["RequestType"] == 'TaskChain':
for taskNumber in range(1, result['TaskChain'] + 1):
taskName = 'Task%s' % taskNumber
if 'ProcessingString' in result[taskName]:
result[taskName]['ProcessingString'] = "BACKFILL"
if 'AcquisitionEra' in result[taskName]:
result[taskName]['AcquisitionEra'] += "Backfill"
# DataProcessing requests don't support RequestNumEvents argument anymore
if 'InputDataset' in result and result['InputDataset']:
result.pop('RequestNumEvents', None)
# Dirty check in TaskChain and StepChains
if 'Task1' in result and 'InputDataset' in result['Task1'] and result[
'Task1']['InputDataset']:
result['Task1'].pop('RequestNumEvents', None)
if 'Step1' in result and 'InputDataset' in result['Step1'] and result[
'Step1']['InputDataset']:
result['Step1'].pop('RequestNumEvents', None)
return result
def modifySchema(cache, workflow, user, group, events, firstLumi, backfill=False, memory=None, timeperevent=None):
"""
Adapts schema to right parameters.
If the original workflow points to DBS2, DBS3 URL is fixed instead.
if backfill is True, modifies RequestString, ProcessingString, AcquisitionEra
and Campaign to say Backfill, and restarts requestDate.
"""
result = reqMgrClient.purgeClonedSchema( cache )
## then further drop nested arguments
taskParamBlacklist = [ 'EventsPerJob' ]
for i in range(1,100):
t='Task%s'%i
if not t in result: break
for p in taskParamBlacklist:
if p in result[t]:
result[t].pop( p )
if memory:
result['Memory'] = memory
if timeperevent:
result['TimePerEvent'] = timeperevent
if user:
result['Requestor'] = user
if group:
result['Group'] = group
# and required for cross-cloning between prod and testbed
if result.get('CouchURL'):
result['ConfigCacheUrl'] = result.pop('CouchURL')
#if we are extending the workflow
if events:
# extend workflow so it will safely start outside of the boundary
RequestNumEvents = int(result['RequestNumEvents'])
FirstEvent = int(result['FirstEvent'])
# FirstEvent_NEW > FirstEvent + RequestNumEvents
# the fist event needs to be oustide the range
result['FirstEvent'] = FirstEvent + RequestNumEvents + DELTA_EVENTS
# FirstLumi_NEW > FirstLumi + RequestNumEvents/events_per_job/filterEff
# same for the first lumi, needs to be after the last lumi
result['FirstLumi'] = firstLumi + DELTA_LUMIS
# only the desired events
result['RequestNumEvents'] = events
# prepend EXT_ to recognize as extension
result["RequestString"] = 'EXT_' + result["RequestString"]
else:
try:
#result["ProcessingVersion"] = int(helper.getProcessingVersion()) + 1
result["ProcessingVersion"] += 1
except ValueError:
result["ProcessingVersion"] = 2
# modify for backfill
if backfill:
# Modify ProcessingString, AcquisitionEra, Campaign and Request string (if they don't
# have the word 'backfill' in it
result["ProcessingString"] = "BACKFILL"
if isinstance(result["AcquisitionEra"],dict):
for eraName in result["AcquisitionEra"]:
if "backfill" not in result["AcquisitionEra"][eraName].lower():
result["AcquisitionEra"][eraName] = result["AcquisitionEra"][eraName] + "Backfill"
else:
if "backfill" not in result["AcquisitionEra"].lower():
result["AcquisitionEra"] = result["AcquisitionEra"] + "Backfill"
if "backfill" not in result["Campaign"].lower():
result["Campaign"] = result["Campaign"] + "-Backfill"
if "backfill" not in result["RequestString"].lower():
# Word backfill in the middle of the request strin
parts = result["RequestString"].split('-')
result["RequestString"] = '-'.join(parts[:2] + ["Backfill"] + parts[2:])
if "PrepID" in result:
# delete entry
del result["PrepID"]
if result["RequestType"] == 'TaskChain':
for taskNumber in range(1, result['TaskChain'] + 1):
taskName = 'Task%s' % taskNumber
if 'ProcessingString' in result[taskName]:
result[taskName]['ProcessingString'] = "BACKFILL"
if 'AcquisitionEra' in result[taskName]:
result[taskName]['AcquisitionEra'] += "Backfill"
# DataProcessing requests don't support RequestNumEvents argument anymore
if 'InputDataset' in result and result['InputDataset']:
result.pop('RequestNumEvents', None)
# Dirty check in TaskChain and StepChains
if 'Task1' in result and 'InputDataset' in result['Task1'] and result['Task1']['InputDataset']:
result['Task1'].pop('RequestNumEvents', None)
if 'Step1' in result and 'InputDataset' in result['Step1'] and result['Step1']['InputDataset']:
result['Step1'].pop('RequestNumEvents', None)
return result
def rejector(url, specific, options=None):
up = componentInfo(soft=['wtc','jira'])
if not up.check(): return
if specific and specific.startswith('/'):
## this is for a dataset
print setDatasetStatus(specific, 'INVALID')
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend( session.query(Workflow).filter(Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(Workflow.name.contains(specific)).all()
if not wfs:
batches = batchInfo().content()
for bname in batches:
if specific == bname:
for pid in batches[bname]:
b_wfs = getWorkflowById(url, pid)
for wf in b_wfs:
wfs.append( session.query(Workflow).filter(Workflow.name == wf).first())
break
else:
wfs = session.query(Workflow).filter(Workflow.status == 'assistance-clone').all()
#wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
## be careful then on clone case by case
options.clone = True
print "not supposed to function yet"
return
print len(wfs),"to reject"
if len(wfs)>1:
print "\n".join( [wfo.name for wfo in wfs] )
answer = raw_input('Reject these')
if not answer.lower() in ['y','yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject",spec
return
wfi = workflowInfo(url, wfo.name)
comment=""
if options.comments: comment = ", reason: "+options.comments
if options.keep:
wfi.sendLog('rejector','invalidating the workflow by unified operator%s'%comment)
else:
wfi.sendLog('rejector','invalidating the workflow and outputs by unified operator%s'%comment)
results = invalidate(url, wfi, only_resub=True, with_output= (not options.keep))
if all(results):
print wfo.name,"rejected"
if options and options.clone:
wfo.status = 'trouble'
session.commit()
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(schema['ProcessingVersion'])+1 ## dubious str->int conversion
else:
schema['ProcessingVersion']=2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
if schema['RequestType'] == 'TaskChain':
it=1
while True:
t = 'Task%d'%it
it+=1
if t in schema:
schema[t]['Memory'] = options.Memory
else:
break
else:
schema['Memory'] = options.Memory
if options.Multicore:
## to do : set it properly in taskchains
if schema['RequestType'] == 'TaskChain':
tasks,set_to = options.Multicore.split(':') if ':' in options.Multicore else ("",options.Multicore)
set_to = int(set_to)
tasks = tasks.split(',') if tasks else ['Task1']
it = 1
while True:
tt = 'Task%d'% it
it+=1
if tt in schema:
tname = schema[tt]['TaskName']
if tname in tasks or tt in tasks:
mem = schema[tt]['Memory']
mcore = schema[tt].get('Multicore',1)
factor = (set_to / float(mcore))
fraction_constant = 0.4
mem_per_core_c = int((1-fraction_constant) * mem / float(mcore))
print "mem per core", mem_per_core_c
print "base mem", mem
## adjusting the parameter in the clone
schema[tt]['Memory'] += (set_to-mcore)*mem_per_core_c
schema[tt]['Multicore'] = set_to
schema[tt]['TimePerEvent'] /= factor
else:
break
else:
schema['Multicore'] = options.Multicore
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int,options.runs.split(','))
if options.PrepID:
schema['PrepID'] =options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1,ntask-1):
schema['Task%d'%it]['KeepOutput'] = False
schema['TaskChain'] = ntask-1
schema.pop('Task%d'%ntask)
if options.priority:
schema['RequestPriority'] = options.priority
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
schema = reqMgrClient.purgeClonedSchema( schema )
print "submitting"
if (options.to_stepchain and (schema['RequestType']=='TaskChain')):
## transform the schema into StepChain schema
print "Transforming a TaskChain into a StepChain"
mcore = 0
mem = 0
schema['RequestType'] = 'StepChain'
schema['StepChain'] = schema.pop('TaskChain')
schema['SizePerEvent'] = 0
schema['TimePerEvent'] = 0
step=1
s_n = {}
while True:
if 'Task%d'%step in schema:
sname = 'Step%d'%step
schema[sname] = schema.pop('Task%d'%step)
tmcore = schema[sname].pop('Multicore')
tmem = schema[sname].pop('Memory')
if mcore and tmcore != mcore:
wfi.sendLog('rejector','the conversion to stepchain encoutered different value of Multicore %d != %d'%( tmcore, mcore))
sendLog('rejector','the conversion of %s to stepchain encoutered different value of Multicore %d != %d'%( wfo.name, tmcore, mcore), level='critical')
mcore = max(mcore, tmcore)
mem = max(mem, tmem)
schema[sname]['StepName'] = schema[sname].pop('TaskName')
s_n[ schema[sname]['StepName'] ] = sname
if 'InputTask' in schema[sname]:
schema[sname]['InputStep'] = schema[sname].pop('InputTask')
eff = 1.
up_s = sname
while True:
## climb up a step. supposedely already all converted
up_s = s_n.get(schema[up_s].get('InputStep',None),None)
if up_s:
## multiply with the efficiency
eff *= schema[up_s].get('FilterEfficiency',1.)
else:
## or stop there
break
if not 'KeepOutput' in schema[sname]:
## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
schema[sname]['KeepOutput'] = False
schema['TimePerEvent'] += eff*schema[sname].pop('TimePerEvent')
schema['SizePerEvent'] += eff*schema[sname].pop('SizePerEvent')
step+=1
else:
break
schema['Multicore'] = mcore
schema['Memory'] = mem
print json.dumps( schema, indent=2 )
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
msg = "Error in cloning {}".format(wfo.name)
print(msg)
wfi.sendLog('rejector',msg)
# Get the error message
time.sleep(5)
data = reqMgrClient.requestManagerPost(url, "/reqmgr2/data/request", schema)
wfi.sendLog('rejector',data)
print json.dumps( schema, indent=2 )
return
print newWorkflow
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfi.sendLog('rejector','Cloned into %s by unified operator %s'%( newWorkflow, comment ))
wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False)
else:
wfo.status = 'trouble' if options.set_trouble else 'forget'
wfi.notifyRequestor('Rejected by unified operator %s'%( comment ),do_batch=False)
session.commit()
else:
msg = "Error in rejecting {}: {}".format(wfo.name,results)
print(msg)
wfi.sendLog('rejector',msg)
def rejector(url, specific, options=None):
up = componentInfo(soft=['wtc'])
if not up.check(): return
if specific and specific.startswith('/'):
## this is for a dataset
print setDatasetStatus(specific, 'INVALID')
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend(
session.query(Workflow).filter(
Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(
Workflow.name.contains(specific)).all()
if not wfs:
batches = json.loads(open('batches.json').read())
for bname in batches:
if specific == bname:
for wf in batches[bname]:
wfs.append(
session.query(Workflow).filter(
Workflow.name == wf).first())
else:
wfs = session.query(Workflow).filter(
Workflow.status == 'assistance-clone').all()
#wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
## be careful then on clone case by case
options.clone = True
print "not supposed to function yet"
return
print len(wfs), "to reject"
if len(wfs) > 1:
print "\n".join([wfo.name for wfo in wfs])
answer = raw_input('Reject these')
if not answer.lower() in ['y', 'yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject", spec
return
results = []
wfi = workflowInfo(url, wfo.name)
datasets = set(wfi.request['OutputDatasets'])
reqMgrClient.invalidateWorkflow(
url, wfo.name, current_status=wfi.request['RequestStatus'])
comment = ""
if options.comments: comment = ", reason: " + options.comments
wfi.sendLog(
'rejector',
'invalidating the workflow by unified operator%s' % comment)
## need to find the whole familly and reject the whole gang
familly = getWorkflowById(url, wfi.request['PrepID'], details=True)
for fwl in familly:
if fwl['RequestDate'] < wfi.request['RequestDate']: continue
if fwl['RequestType'] != 'Resubmission': continue
## does not work on second order acd
#if 'OriginalRequestName' in fwl and fwl['OriginalRequestName'] != wfi.request['RequestName']: continue
print "rejecting", fwl['RequestName']
reqMgrClient.invalidateWorkflow(
url,
fwl['RequestName'],
current_status=fwl['RequestStatus'],
cascade=False)
datasets.update(fwl['OutputDatasets'])
for dataset in datasets:
if options.keep:
print "keeping", dataset, "in its current status"
else:
results.append(setDatasetStatus(dataset, 'INVALID'))
pass
if all(map(lambda result: result in ['None', None, True], results)):
print wfo.name, "and", datasets, "are rejected"
if options and options.clone:
wfo.status = 'trouble'
session.commit()
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(
schema['ProcessingVersion']
) + 1 ## dubious str->int conversion
else:
schema['ProcessingVersion'] = 2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
if schema['RequestType'] == 'TaskChain':
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in schema:
schema[t]['Memory'] = options.Memory
else:
break
else:
schema['Memory'] = options.Memory
if options.Multicore:
## to do : set it properly in taskchains
if schema['RequestType'] == 'TaskChain':
tasks, set_to = options.Multicore.split(
':') if ':' in options.Multicore else (
"", options.Multicore)
set_to = int(set_to)
tasks = tasks.split(',') if tasks else ['Task1']
it = 1
while True:
tt = 'Task%d' % it
it += 1
if tt in schema:
tname = schema[tt]['TaskName']
if tname in tasks or tt in tasks:
mem = schema[tt]['Memory']
mcore = schema[tt].get('Multicore', 1)
factor = (set_to / float(mcore))
fraction_constant = 0.4
mem_per_core_c = int(
(1 - fraction_constant) * mem /
float(mcore))
print "mem per core", mem_per_core_c
print "base mem", mem
## adjusting the parameter in the clone
schema[tt]['Memory'] += (
set_to - mcore) * mem_per_core_c
schema[tt]['Multicore'] = set_to
schema[tt]['TimePerEvent'] /= factor
else:
break
else:
schema['Multicore'] = options.Multicore
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int, options.runs.split(','))
if options.PrepID:
schema['PrepID'] = options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1, ntask - 1):
schema['Task%d' % it]['KeepOutput'] = False
schema['TaskChain'] = ntask - 1
schema.pop('Task%d' % ntask)
if options.priority:
schema['RequestPriority'] = options.priority
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
schema = reqMgrClient.purgeClonedSchema(schema)
print "submitting"
if (options.to_stepchain
and (schema['RequestType'] == 'TaskChain')):
## transform the schema into StepChain schema
print "Transforming a TaskChain into a StepChain"
schema['RequestType'] = 'StepChain'
schema['StepChain'] = schema.pop('TaskChain')
schema['SizePerEvent'] = 0
schema['TimePerEvent'] = 0
step = 1
s_n = {}
while True:
if 'Task%d' % step in schema:
schema['Step%d' % step] = schema.pop('Task%d' %
step)
schema['Step%d' % step]['StepName'] = schema[
'Step%d' % step].pop('TaskName')
s_n[schema['Step%d' %
step]['StepName']] = 'Step%d' % step
if 'InputTask' in schema['Step%d' % step]:
schema['Step%d' % step]['InputStep'] = schema[
'Step%d' % step].pop('InputTask')
eff = 1.
up_s = 'Step%d' % step
while True:
## climb up a step. supposedely already all converted
up_s = s_n.get(
schema[up_s].get('InputStep', None), None)
if up_s:
## multiply with the efficiency
eff *= schema[up_s].get(
'FilterEfficiency', 1.)
else:
## or stop there
break
if not 'KeepOutput' in schema['Step%d' % step]:
## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
schema['Step%d' % step]['KeepOutput'] = False
schema['TimePerEvent'] += eff * schema[
'Step%d' % step].pop('TimePerEvent')
schema['SizePerEvent'] += eff * schema[
'Step%d' % step].pop('SizePerEvent')
step += 1
else:
break
print json.dumps(schema, indent=2)
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
print "error in cloning", wfo.name
print json.dumps(schema, indent=2)
return
print newWorkflow
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfi.sendLog(
'rejector', 'Cloned into %s by unified operator %s' %
(newWorkflow, comment))
wfi.notifyRequestor('Cloned into %s by unified operator %s' %
(newWorkflow, comment),
do_batch=False)
else:
wfo.status = 'trouble' if options.set_trouble else 'forget'
wfi.notifyRequestor('Rejected by unified operator %s' %
(comment),
do_batch=False)
session.commit()
else:
print "error in rejecting", wfo.name, results
def singleClone(url, wfname, actions, comment, do=False):
wfi = workflowInfo(url, wfname)
payload = wfi.getSchema()
initial = wfi.request
payload['Requestor'] = os.getenv('USER')
payload['Group'] = 'DATAOPS'
payload['OriginalRequestName'] = initial['RequestName']
payload['RequestPriority'] = initial['RequestPriority']
if 'ProcessingVersion' in initial:
payload['ProcessingVersion'] = int(initial['ProcessingVersion']) +1
else:
payload['ProcessingVersion'] = 2
payload = reqMgrClient.purgeClonedSchema( payload )
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
if 'TaskChain' in payload:
print "Setting memory for clone of task chain"
mem_dict = {}
it=1
while True:
t = 'Task%d'%it
it+=1
if t in payload:
tname = payload[t]['TaskName']
mem_dict[tname] = int(actions[action])
print "Memory set for Task%d"%it
else:
break
payload['Memory'] = mem_dict
else:
print "Setting memory for non-taskchain workflow"
payload['Memory'] = int(actions[action])
print "Memory set to " + actions[action]
print "Clone payload"
# print json.dumps( payload , indent=2)
print actions
#Create clone
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error in making clone for",initial["RequestName"]
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error twice in making clone for",initial["RequestName"]
sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"],level='critical')
wfi.sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"])
return None
if actions:
for action in actions:
if action.startswith('split'):
cloneinfo = workflowInfo(url, clone)
splittings = cloneinfo.getSplittingsNew(strip=True)
if actions[action] != 'Same' and actions[action] != 'max' and actions[action] != '':
factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
split_par = split['splitParams']
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split_par:
wfi.sendLog('actor','Changing %s (%d) by a factor %d'%( act, split_par[act], factor))
split_par[act] /= factor
print "to",split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
elif 'max' in actions[action]:
for split in splittings:
split_par = split['splitParams']
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split_par:
wfi.sendLog('actor','Max splitting set for %s (%d'%( act, split_par[act]))
print "Changing %s (%d) "%( act, split_par[act]),
split_par[act] = 1
print "to max splitting ",split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
print "changing the splitting of",clone
print json.dumps( splittings, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, clone, splittings )
#Approve
data = reqMgrClient.setWorkflowApproved(url, clone)
#wfi.sendLog('actor','Cloned into %s'%clone)
# wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
# wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)
print data
return clone