def cloneWorkflow(workflow, user, group, verbose=True, backfill=False, testbed=False, memory=None, bwl=None):
"""
clones a workflow
"""
# Get info about the workflow to be cloned
helper = reqMgrClient.retrieveSchema(workflow, reqmgrCouchURL)
# Adapt schema and add original request to it
try:
cache = reqMgrClient.getWorkloadCache(url, workflow)
except:
cache = None
schema = modifySchema(helper, workflow, user, group, cache, None, None, backfill, memory)
schema['OriginalRequestName'] = workflow
if verbose:
pprint(schema)
if bwl:
if 'Task1' in schema:
schema['Task1']['BlockWhitelist'] = bwl.split(',')
else:
schema['BlockWhitelist'] = bwl.split(',')
print 'Submitting workflow'
# Submit cloned workflow to ReqMgr
if testbed:
newWorkflow = reqMgrClient.submitWorkflow(url_tb, schema)
else:
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if verbose:
print "RESPONSE", newWorkflow
# find the workflow name in response
if newWorkflow:
print 'Cloned workflow: ' + newWorkflow
if verbose:
print newWorkflow
print 'Approving request response:'
# TODO only for debug
#response = reqMgrClient.setWorkflowSplitting(url, schema)
# print "RESPONSE", response
#schema['requestName'] = requestName
#schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
#schema['splittingAlgo'] = splittingAlgo
# Move the request to Assignment-approved
if testbed:
data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
else:
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
if verbose:
print data
# return the name of new workflow
return newWorkflow
else:
if verbose:
print newWorkflow
else:
print "Couldn't clone the workflow."
return None
def cloneWorkflow(workflow, user, group, verbose=True, backfill=False, testbed=False, memory=None, bwl=None):
"""
clones a workflow
"""
# Get info about the workflow to be cloned
helper = reqMgrClient.retrieveSchema(workflow, reqmgrCouchURL)
# Adapt schema and add original request to it
try:
cache = reqMgrClient.getWorkloadCache(url, workflow)
except:
cache = None
schema = modifySchema(helper, workflow, user, group, cache, None, None, backfill, memory)
schema['OriginalRequestName'] = workflow
if verbose:
pprint(schema)
if bwl:
if 'Task1' in schema:
schema['Task1']['BlockWhitelist'] = bwl.split(',')
else:
schema['BlockWhitelist'] = bwl.split(',')
print 'Submitting workflow'
# Submit cloned workflow to ReqMgr
if testbed:
newWorkflow = reqMgrClient.submitWorkflow(url_tb, schema, reqmgr2 = True)
else:
newWorkflow = reqMgrClient.submitWorkflow(url, schema, reqmgr2 = True)
if verbose:
print "RESPONSE", newWorkflow
# find the workflow name in response
if newWorkflow:
print 'Cloned workflow: ' + newWorkflow
if verbose:
print newWorkflow
print 'Approving request response:'
# TODO only for debug
#response = reqMgrClient.setWorkflowSplitting(url, schema)
# print "RESPONSE", response
#schema['requestName'] = requestName
#schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
#schema['splittingAlgo'] = splittingAlgo
# Move the request to Assignment-approved
if testbed:
data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
else:
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
if verbose:
print data
# return the name of new workflow
return newWorkflow
else:
if verbose:
print newWorkflow
else:
print "Couldn't clone the workflow."
return None
def cloneWorkflow(workflow, user, group, verbose=True, backfill=False, testbed=False, memory=None, timeperevent=None, bwl=None):
"""
clones a workflow
"""
# Adapt schema and add original request to it
cache = reqMgrClient.getWorkflowInfo(url, workflow)
schema = modifySchema(cache, workflow, user, group, None, None, backfill, memory, timeperevent)
if verbose:
pprint(schema)
if bwl:
if 'Task1' in schema:
schema['Task1']['BlockWhitelist'] = bwl.split(',')
else:
schema['BlockWhitelist'] = bwl.split(',')
## only once
####schema['CMSSWVersion'] = 'CMSSW_8_0_16'
print 'Submitting workflow'
# Submit cloned workflow to ReqMgr
if testbed:
newWorkflow = reqMgrClient.submitWorkflow(url_tb, schema)
else:
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if verbose:
print "RESPONSE", newWorkflow
# find the workflow name in response
if newWorkflow:
print 'Cloned workflow: ' + newWorkflow
if verbose:
print newWorkflow
print 'Approving request response:'
# TODO only for debug
#response = reqMgrClient.setWorkflowSplitting(url, schema)
# print "RESPONSE", response
#schema['requestName'] = requestName
#schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
#schema['splittingAlgo'] = splittingAlgo
# Move the request to Assignment-approved
if testbed:
data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
else:
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
if verbose:
print data
# return the name of new workflow
return newWorkflow
else:
if verbose:
print newWorkflow
else:
print "Couldn't clone the workflow."
return None
def cloneWorkflow(workflow, user, group, verbose=False, backfill=False, testbed=False):
"""
clones a workflow
"""
# Get info about the workflow to be cloned
helper = reqMgrClient.retrieveSchema(workflow, reqmgrCouchURL)
# Adapt schema and add original request to it
try:
cache = reqMgrClient.getWorkloadCache(url, workflow)
except:
cache = None
schema = modifySchema(helper, user, group, cache, backfill)
schema['OriginalRequestName'] = workflow
if verbose:
pprint(schema)
print 'Submitting workflow'
# Sumbit cloned workflow to ReqMgr
if testbed:
response = reqMgrClient.submitWorkflow(url_tb, schema)
else:
response = reqMgrClient.submitWorkflow(url, schema)
if verbose:
print "RESPONSE", response
# find the workflow name in response
m = re.search("details\/(.*)\'", response)
if m:
newWorkflow = m.group(1)
print 'Cloned workflow: ' + newWorkflow
if verbose:
print response
print 'Approving request response:'
# TODO only for debug
#response = reqMgrClient.setWorkflowSplitting(url, schema)
# print "RESPONSE", response
#schema['requestName'] = requestName
#schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
#schema['splittingAlgo'] = splittingAlgo
# Move the request to Assignment-approved
if testbed:
data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
else:
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
if verbose:
print data
# return the name of new workflow
return newWorkflow
else:
if verbose:
print response
else:
print "Couldn't clone the workflow."
return None
def cloneWorkflow(workflow, user, group, verbose=False):
"""
clones a workflow
"""
# Get info about the workflow to be cloned
helper = retrieveSchema(workflow)
schema = modifySchema(helper, user, group)
print 'Submitting workflow'
# Sumbit cloned workflow to ReqMgr
response = reqMgrClient.submitWorkflow(url,schema)
#find the workflow name in response
m = re.search("details\/(.*)\'",response)
if m:
newWorkflow = m.group(1)
if verbose:
print 'Cloned workflow: '+newWorkflow
print response
print 'Approve request response:'
# Move the request to Assignment-approved
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
if verbose:
print data
#return the name of new workflow
return newWorkflow
else:
if verbose:
print response
return None
def rejector(url, specific, options=None):
if specific.startswith('/'):
pass
else:
wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject", spec
return
results = []
wfi = workflowInfo(url, wfo.name)
if wfi.request['RequestStatus'] in [
'assignment-approved', 'new', 'completed'
]:
#results.append( reqMgrClient.rejectWorkflow(url, wfo.name))
reqMgrClient.rejectWorkflow(url, wfo.name)
else:
#results.append( reqMgrClient.abortWorkflow(url, wfo.name))
reqMgrClient.abortWorkflow(url, wfo.name)
datasets = wfi.request['OutputDatasets']
for dataset in datasets:
if options.keep:
print "keeping", dataset, "in its current status"
else:
results.append(
setDatasetStatusDBS3.setStatusDBS3(
'https://cmsweb.cern.ch/dbs/prod/global/DBSWriter',
dataset, 'INVALID', None))
if all(map(lambda result: result in ['None', None], results)):
wfo.status = 'forget'
session.commit()
print wfo.name, "and", datasets, "are rejected"
if options and options.clone:
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] += 1
else:
schema['ProcessingVersion'] = 2
##schema.pop('RequestDate') ## ok then, let's not reset the time stamp
if options.Memory:
schema['Memory'] = options.Memory
response = reqMgrClient.submitWorkflow(url, schema)
m = re.search("details\/(.*)\'", response)
if not m:
print "error in cloning", wfo.name
print response
return
newWorkflow = m.group(1)
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfo.status = 'trouble'
session.commit()
else:
print "error in rejecting", wfo.name, results
def rejector(url, specific, options=None):
up = componentInfo()
if specific.startswith('/'):
pass
else:
wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject",spec
return
results=[]
wfi = workflowInfo(url, wfo.name)
reqMgrClient.invalidateWorkflow(url, wfo.name, current_status=wfi.request['RequestStatus'])
#if wfi.request['RequestStatus'] in ['assignment-approved','new','completed']:
# #results.append( reqMgrClient.rejectWorkflow(url, wfo.name))
# reqMgrClient.rejectWorkflow(url, wfo.name)
#else:
# #results.append( reqMgrClient.abortWorkflow(url, wfo.name))
# reqMgrClient.abortWorkflow(url, wfo.name)
datasets = wfi.request['OutputDatasets']
for dataset in datasets:
if options.keep:
print "keeping",dataset,"in its current status"
else:
results.append( setDatasetStatus(dataset, 'INVALID') )
if all(map(lambda result : result in ['None',None,True],results)):
wfo.status = 'forget'
session.commit()
print wfo.name,"and",datasets,"are rejected"
if options and options.clone:
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion']+=1
else:
schema['ProcessingVersion']=2
##schema.pop('RequestDate') ## ok then, let's not reset the time stamp
if options.Memory:
schema['Memory'] = options.Memory
response = reqMgrClient.submitWorkflow(url, schema)
m = re.search("details\/(.*)\'",response)
if not m:
print "error in cloning",wfo.name
print response
return
newWorkflow = m.group(1)
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfo.status = 'trouble'
session.commit()
else:
print "error in rejecting",wfo.name,results
def extendWorkflow(workflow,
user,
group,
verbose=False,
events=None,
firstlumi=None):
if events is None:
events = getMissingEvents(workflow)
events = int(events)
if firstlumi is None:
#get the last lumi of the dataset
dataset = reqMgrClient.outputdatasetsWorkflow(url, workflow).pop()
lastLumi = dbs3Client.getMaxLumi(dataset)
firstlumi = lastLumi
firstlumi = int(firstlumi)
# Get info about the workflow to be cloned
cache = reqMgrClient.getWorkflowInfo(url, workflow)
schema = modifySchema(cache, workflow, user, group, events, firstlumi,
None)
if verbose:
pprint(schema)
print 'Submitting workflow'
# Submit cloned workflow to ReqMgr
response = reqMgrClient.submitWorkflow(url, schema)
if verbose:
print "RESPONSE", response
#find the workflow name in response
m = re.search("details\/(.*)\'", response)
if m:
newWorkflow = m.group(1)
print 'Cloned workflow: ' + newWorkflow
print 'Extended with', events, 'events'
print response
# Move the request to Assignment-approved
print 'Approve request response:'
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
else:
print response
def cloneWorkflow(workflow, user, group, verbose=False, backfill=False):
"""
clones a workflow
"""
# Get info about the workflow to be cloned
helper = reqMgrClient.retrieveSchema(workflow)
# get info from reqMgr
schema = modifySchema(helper, user, group, backfill)
if verbose:
pprint(schema)
print 'Submitting workflow'
# Sumbit cloned workflow to ReqMgr
response = reqMgrClient.submitWorkflow(url,schema)
if verbose:
print "RESPONSE", response
#find the workflow name in response
m = re.search("details\/(.*)\'",response)
if m:
newWorkflow = m.group(1)
print 'Cloned workflow: '+newWorkflow
if verbose:
print response
print 'Approving request response:'
#TODO only for debug
#response = reqMgrClient.setWorkflowSplitting(url, schema)
#print "RESPONSE", response
#schema['requestName'] = requestName
#schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
#schema['splittingAlgo'] = splittingAlgo
# Move the request to Assignment-approved
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
if verbose:
print data
#return the name of new workflow
return newWorkflow
else:
if verbose:
print response
else:
print "Couldn't clone the workflow."
return None
def cloneWorkflow(workflow, user, group, verbose=False, backfill=False):
"""
clones a workflow
"""
# Get info about the workflow to be cloned
helper = reqMgrClient.retrieveSchema(workflow, reqmgrCouchURL)
# get info from reqMgr
schema = modifySchema(helper, user, group, backfill)
if verbose:
pprint(schema)
print 'Submitting workflow'
# Sumbit cloned workflow to ReqMgr
response = reqMgrClient.submitWorkflow(url, schema)
if verbose:
print "RESPONSE", response
#find the workflow name in response
m = re.search("details\/(.*)\'", response)
if m:
newWorkflow = m.group(1)
print 'Cloned workflow: ' + newWorkflow
if verbose:
print response
print 'Approving request response:'
#TODO only for debug
#response = reqMgrClient.setWorkflowSplitting(url, schema)
#print "RESPONSE", response
#schema['requestName'] = requestName
#schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
#schema['splittingAlgo'] = splittingAlgo
# Move the request to Assignment-approved
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
if verbose:
print data
#return the name of new workflow
return newWorkflow
else:
if verbose:
print response
else:
print "Couldn't clone the workflow."
return None
def extendWorkflow(workflow, user, group, verbose=False, events=None, firstlumi=None):
if events is None:
events = getMissingEvents(workflow)
events = int(events)
if firstlumi is None:
#get the last lumi of the dataset
dataset = reqMgrClient.outputdatasetsWorkflow(url, workflow).pop()
lastLumi = dbs3Client.getMaxLumi(dataset)
firstlumi = lastLumi
firstlumi = int(firstlumi)
# Get info about the workflow to be cloned
helper = reqMgrClient.retrieveSchema(workflow)
schema = modifySchema(helper, workflow, user, group, events, firstlumi)
schema['OriginalRequestName'] = workflow
if verbose:
pprint(schema)
print 'Submitting workflow'
# Sumbit cloned workflow to ReqMgr
response = reqMgrClient.submitWorkflow(url,schema)
if verbose:
print "RESPONSE", response
#find the workflow name in response
m = re.search("details\/(.*)\'",response)
if m:
newWorkflow = m.group(1)
print 'Cloned workflow: '+newWorkflow
print 'Extended with', events, 'events'
print response
# Move the request to Assignment-approved
print 'Approve request response:'
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
else:
print response
pass
def main():
# Check the arguements, get info from them
if len(sys.argv) < 4:
print "Usage:"
print " ./resubmit WORKFLOW_NAME USER GROUP [EVENTS]"
sys.exit(0)
workflow = sys.argv[1]
user = sys.argv[2]
group = sys.argv[3]
if len(sys.argv) > 4:
events = int(sys.argv[4])
else:
events = getMissingEvents(workflow)
# Get info about the workflow to be cloned
helper = reqMgrClient.retrieveSchema(workflow)
schema = modifySchema(helper, workflow, user, group, events)
print 'Submitting workflow'
# Sumbit cloned workflow to ReqMgr
response = reqMgrClient.submitWorkflow(url,schema)
#find the workflow name in response
m = re.search("details\/(.*)\'",response)
if m:
newWorkflow = m.group(1)
print 'Cloned workflow: '+newWorkflow
print 'Extended with', events, 'events'
print response
# Move the request to Assignment-approved
print 'Approve request response:'
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
else:
print response
sys.exit(0)
def cloneWorkflow(workflow,
user,
group,
verbose=True,
backfill=False,
testbed=False,
memory=None,
timeperevent=None,
bwl=None):
"""
clones a workflow
"""
# Adapt schema and add original request to it
cache = reqMgrClient.getWorkflowInfo(url, workflow)
schema = modifySchema(cache, workflow, user, group, None, None, backfill,
memory, timeperevent)
if verbose:
pprint(schema)
if bwl:
if 'Task1' in schema:
schema['Task1']['BlockWhitelist'] = bwl.split(',')
else:
schema['BlockWhitelist'] = bwl.split(',')
## only once
####schema['CMSSWVersion'] = 'CMSSW_8_0_16'
print 'Submitting workflow'
# Submit cloned workflow to ReqMgr
if testbed:
newWorkflow = reqMgrClient.submitWorkflow(url_tb, schema)
else:
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if verbose:
print "RESPONSE", newWorkflow
# find the workflow name in response
if newWorkflow:
print 'Cloned workflow: ' + newWorkflow
if verbose:
print newWorkflow
print 'Approving request response:'
# TODO only for debug
#response = reqMgrClient.setWorkflowSplitting(url, schema)
# print "RESPONSE", response
#schema['requestName'] = requestName
#schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
#schema['splittingAlgo'] = splittingAlgo
# Move the request to Assignment-approved
if testbed:
data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
else:
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
if verbose:
print data
# return the name of new workflow
return newWorkflow
else:
if verbose:
print newWorkflow
else:
print "Couldn't clone the workflow."
return None
def rejector(url, specific, options=None):
#use_mcm = True
#up = componentInfo(mcm=use_mcm, soft=['mcm'])
up = componentInfo()
if not up.check(): return
#use_mcm = up.status['mcm']
#mcm = McMClient(dev=False) if use_mcm else None
if specific and specific.startswith('/'):
## this is for a dataset
print setDatasetStatus(specific, 'INVALID')
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend( session.query(Workflow).filter(Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(Workflow.name.contains(specific)).all()
else:
wfs = session.query(Workflow).filter(Workflow.status == 'assistance-clone').all()
#wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
## be careful then on clone case by case
options.clone = True
print "not supposed to function yet"
return
print len(wfs),"to reject"
if len(wfs)>1:
print "\n".join( [wfo.name for wfo in wfs] )
answer = raw_input('Reject these')
if not answer.lower() in ['y','yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject",spec
return
results=[]
wfi = workflowInfo(url, wfo.name)
datasets = set(wfi.request['OutputDatasets'])
reqMgrClient.invalidateWorkflow(url, wfo.name, current_status=wfi.request['RequestStatus'])
comment=""
if options.comments: comment = ", reason: "+options.comments
wfi.sendLog('rejector','invalidating the workflow by unified operator%s'%comment)
## need to find the whole familly and reject the whole gang
familly = getWorkflowById( url, wfi.request['PrepID'] , details=True)
for fwl in familly:
if fwl['RequestDate'] < wfi.request['RequestDate']: continue
if fwl['RequestType']!='Resubmission': continue
## does not work on second order acd
if 'OriginalRequestName' in fwl and fwl['OriginalRequestName'] != wfi.request['RequestName']: continue
print "rejecting",fwl['RequestName']
reqMgrClient.invalidateWorkflow(url, fwl['RequestName'], current_status=fwl['RequestStatus'], cascade=False)
datasets.update( fwl['OutputDatasets'] )
for dataset in datasets:
if options.keep:
print "keeping",dataset,"in its current status"
else:
results.append( setDatasetStatus(dataset, 'INVALID') )
pass
if all(map(lambda result : result in ['None',None,True],results)):
print wfo.name,"and",datasets,"are rejected"
if options and options.clone:
wfo.status = 'trouble'
session.commit()
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(schema['ProcessingVersion'])+1 ## dubious str->int conversion
else:
schema['ProcessingVersion']=2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
if schema['RequestType'] == 'TaskChain':
it=1
while True:
t = 'Task%d'%it
it+=1
if t in schema:
schema[t]['Memory'] = options.Memory
else:
break
else:
schema['Memory'] = options.Memory
if options.Multicore:
## to do : set it properly in taslchains
schema['Multicore'] = options.Multicore
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int,options.runs.split(','))
if options.PrepID:
schema['PrepID'] =options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1,ntask-1):
schema['Task%d'%it]['KeepOutput'] = False
schema['TaskChain'] = ntask-1
schema.pop('Task%d'%ntask)
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
paramBlacklist = ['BlockCloseMaxEvents', 'BlockCloseMaxFiles', 'BlockCloseMaxSize', 'BlockCloseMaxWaitTime',
'CouchWorkloadDBName', 'CustodialGroup', 'CustodialSubType', 'Dashboard',
'GracePeriod', 'HardTimeout', 'InitialPriority', 'inputMode', 'MaxMergeEvents', 'MaxMergeSize',
'MaxRSS', 'MaxVSize', 'MinMergeSize', 'NonCustodialGroup', 'NonCustodialSubType',
'OutputDatasets', 'ReqMgr2Only', 'RequestDate' 'RequestorDN', 'RequestName', 'RequestStatus',
'RequestTransition', 'RequestWorkflow', 'SiteWhitelist', 'SoftTimeout', 'SoftwareVersions',
'SubscriptionPriority', 'Team', 'timeStamp', 'TrustSitelists', 'TrustPUSitelists',
'TotalEstimatedJobs', 'TotalInputEvents', 'TotalInputLumis', 'TotalInputFiles']
for p in paramBlacklist:
if p in schema:
schema.pop( p )
#pass
print "submitting"
if (options.to_stepchain and (schema['RequestType']=='TaskChain')):
## transform the schema into StepChain schema
print "Transforming a TaskChain into a StepChain"
schema['RequestType'] = 'StepChain'
schema['StepChain'] = schema.pop('TaskChain')
step=1
while True:
if 'Task%d'%step in schema:
schema['Step%d'%step] = schema.pop('Task%d'%step)
schema['Step%d'%step]['StepName'] = schema['Step%d'%step].pop('TaskName')
if 'InputTask' in schema['Step%d'%step]:
schema['Step%d'%step]['InputStep'] = schema['Step%d'%step].pop('InputTask')
if not 'KeepOutput' in schema['Step%d'%step]:
## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
schema['Step%d'%step]['KeepOutput'] = False
step+=1
else:
break
print json.dumps( schema, indent=2 )
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
print "error in cloning",wfo.name
print json.dumps( schema, indent=2 )
return
print newWorkflow
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfi.sendLog('rejector','Cloned into %s by unified operator %s'%( newWorkflow, comment ))
wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False)
else:
wfo.status = 'forget'
wfi.notifyRequestor('Rejected by unified operator %s'%( comment ),do_batch=False)
session.commit()
else:
print "error in rejecting",wfo.name,results
def spawn_harvesting(url, wfi , in_full):
SI = global_SI()
all_OK = {}
requests = []
outputs = wfi.request['OutputDatasets']
if ('EnableHarvesting' in wfi.request and not wfi.request['EnableHarvesting']) and ('DQMConfigCacheID' in wfi.request and wfi.request['DQMConfigCacheID']):
if not 'MergedLFNBase' in wfi.request:
print "f****d up"
sendEmail('screwed up wl cache','%s wl cache is bad'%(wfi.request['RequestName']))
all_OK['fake'] = False
return all_OK,requests
wfi = workflowInfo(url, wfi.request['RequestName'])
dqms = [out for out in outputs if '/DQM' in out]
if not all([in_full[dqm_input] for dqm_input in dqms]):
wfi.sendLog('closor',"will not be able to assign the harvesting: holding up")
for dqm_input in dqms:
all_OK[dqm_input] = False
## raise the subscription to high priority
sites = set(wfi.request['NonCustodialSites'])
for site in sites:
res = updateSubscription(url, site, dqm_input, priority='reserved')
print "increased priority",res
## make a subscription somewhere else in the T1s at random each time, hoping it will get there eventually
#pick = SI.CE_to_SE(random.choice(SI.sites_T1s))
#print "replicating",dqm_input,"to",pick
#res = makeReplicaRequest(url, pick, [dqm_input], "Replicating DQM for harvesting", priority='normal', approve=True, mail=False)
return all_OK,requests
for dqm_input in dqms:
## handle it properly
harvesting_schema = {
'Requestor': os.getenv('USER'),
'RequestType' : 'DQMHarvest',
'Group' : 'DATAOPS'
}
copy_over = [
'AcquisitionEra',
'ProcessingString',
'DQMUploadUrl',
'CMSSWVersion',
'CouchDBName',
'CouchWorkloadDBName',
'ConfigCacheUrl',
'DbsUrl',
'inputMode',
'DQMConfigCacheID',
'OpenRunningTimeout',
'ScramArch',
'CMSSWVersion',
'Campaign',
'Memory', #dummy
'SizePerEvent', #dummy
'GlobalTag', #dummy
]
for item in copy_over:
if item in wfi.request:
harvesting_schema[item] = copy.deepcopy(wfi.request[item])
else:
print item,"is not in initial schema"
harvesting_schema['InputDataset'] = dqm_input
harvesting_schema['TimePerEvent'] = 1
harvesting_schema['PrepID'] = 'Harvest-'+wfi.request['PrepID']
if len(wfi.request['RequestString'])>60:
wfi.request['RequestString']= wfi.request['RequestString'][:60]
print "truncating request string",wfi.request['RequestString']
harvesting_schema['RequestString'] = 'HARVEST-'+wfi.request['RequestString']
harvesting_schema['DQMHarvestUnit'] = 'byRun'
harvesting_schema['RequestPriority'] = min(wfi.request['RequestPriority']*10,999999)
harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema)
if not harvest_request:
print "Error in making harvesting for",wfi.request['RequestName']
print "schema"
print json.dumps( harvesting_schema, indent = 2)
harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema)
if not harvest_request:
print "Error twice in harvesting for",wfi.request['RequestName']
print "schema"
print json.dumps( harvesting_schema, indent = 2)
if harvest_request:
requests.append( harvest_request )
## should we protect for setting approved ? no, it's notified below, assignment will fail, likely
data = reqMgrClient.setWorkflowApproved(url, harvest_request)
print "created",harvest_request,"for harvesting of",dqm_input
wfi.sendLog('closor',"created %s for harvesting of %s"%( harvest_request, dqm_input))
## assign it directly
team = wfi.request['Team']
parameters={
'SiteWhitelist' : [SI.SE_to_CE(se) for se in wfi.request['NonCustodialSites']],
'AcquisitionEra' : wfi.acquisitionEra(),
'ProcessingString' : wfi.processingString(),
'MergedLFNBase' : wfi.request['MergedLFNBase'],
'ProcessingVersion' : wfi.request['ProcessingVersion'],
'execute' : True
}
if in_full[dqm_input]:
print "using full copy at",in_full[dqm_input]
parameters['SiteWhitelist'] = [SI.SE_to_CE(se) for se in in_full[dqm_input]]
else:
print "cannot do anything if not having a full copy somewhere"
all_OK[dqm_input]=False
continue
result = reqMgrClient.assignWorkflow(url, harvest_request, team, parameters)
if not result:
#sendEmail('harvesting request created','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
wfi.sendLog('closor','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']))
sendLog('closor','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), level='critical')
else:
#sendEmail('harvesting request assigned','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
wfi.sendLog('closor','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']))
else:
#print "could not make the harvesting for",wfo.name,"not announcing"
wfi.sendLog('closor',"could not make the harvesting request")
sendLog('closor',"could not make the harvesting request for %s"% wfi.request['RequestName'], level='critical')
all_OK[dqm_input]=False
return (all_OK, requests)
def rejector(url, specific, options=None):
up = componentInfo()
if specific and specific.startswith('/'):
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend(
session.query(Workflow).filter(
Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(
Workflow.name.contains(specific)).all()
else:
return
print len(wfs), "to reject"
if len(wfs) > 1:
print "\n".join([wfo.name for wfo in wfs])
answer = raw_input('Reject these')
if not answer.lower() in ['y', 'yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject", spec
return
results = []
wfi = workflowInfo(url, wfo.name)
datasets = set(wfi.request['OutputDatasets'])
reqMgrClient.invalidateWorkflow(
url, wfo.name, current_status=wfi.request['RequestStatus'])
wfi.sendLog('rejector',
'invalidating the workflow by unified operator')
## need to find the whole familly and reject the whole gang
familly = getWorkflowById(url, wfi.request['PrepID'], details=True)
for fwl in familly:
if fwl['RequestDate'] < wfi.request['RequestDate']: continue
if fwl['RequestType'] != 'Resubmission': continue
if 'OriginalRequestName' in fwl and fwl[
'OriginalRequestName'] != wfi.request['RequestName']:
continue
print "rejecting", fwl['RequestName']
reqMgrClient.invalidateWorkflow(
url, fwl['RequestName'], current_status=fwl['RequestStatus'])
datasets.update(fwl['OutputDatasets'])
for dataset in datasets:
if options.keep:
print "keeping", dataset, "in its current status"
else:
results.append(setDatasetStatus(dataset, 'INVALID'))
#if wfi.request['RequestStatus'] in ['rejected','rejected-archived','aborted','aborted-archived']:
# print 'already',wfi.request['RequestStatus']
# if not options.clone:
# wfo.status = 'forget'
# session.commit()
# continue
if all(map(lambda result: result in ['None', None, True], results)):
wfo.status = 'forget'
session.commit()
print wfo.name, "and", datasets, "are rejected"
if options and options.clone:
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(
schema['ProcessingVersion']
) + 1 ## dubious str->int conversion
else:
schema['ProcessingVersion'] = 2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
schema['Memory'] = options.Memory
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int, options.runs.split(','))
if options.PrepID:
schema['PrepID'] = options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1, ntask - 1):
schema['Task%d' % it]['KeepOutput'] = False
schema['TaskChain'] = ntask - 1
schema.pop('Task%d' % ntask)
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
for p in [
'RequestStatus',
'RequestTransition',
'RequestorDN',
'RequestWorkflow',
'OutputDatasets',
'ReqMgr2Only',
#'Group',
'RequestDate',
#'ConfigCacheUrl',
'RequestName',
'timeStamp',
'SoftwareVersions',
'CouchURL'
]:
if p in schema:
schema.pop(p)
#pass
print "submitting"
print json.dumps(schema, indent=2)
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
print "error in cloning", wfo.name
print json.dumps(schema, indent=2)
return
print newWorkflow
#m = re.search("details\/(.*)\'",response)
#if not m:
# print "error in cloning",wfo.name
# print response
# print json.dumps( schema, indent=2 )
# return
#newWorkflow = m.group(1)
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfo.status = 'trouble'
session.commit()
wfi.sendLog(
'rejector',
'Cloned into %s by unified operator' % (newWorkflow))
else:
print "error in rejecting", wfo.name, results
def singleRecovery(url, task, initial, actions, do=False):
print "Inside single recovery!"
payload = {
"Requestor" : os.getenv('USER'),
"Group" : 'DATAOPS',
"RequestType" : "Resubmission",
"ACDCServer" : initial['CouchURL'],
"ACDCDatabase" : "acdcserver",
"OriginalRequestName" : initial['RequestName'],
"OpenRunningTimeout" : 0
}
copy_over = ['PrepID','Campaign','RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ,'CMSSWVersion']
for c in copy_over:
if c in initial:
payload[c] = copy.deepcopy(initial[c])
else:
print c,"not in the initial payload"
#a massage ? boost the recovery over the initial wf
# payload['RequestPriority'] *= 10
#Max priority is 1M
payload['RequestPriority'] = min(500000, payload['RequestPriority']*2 ) ## never above 500k
#change parameters based on actions here
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
payload['Memory'] = actions[action]
print "Memory set to " + actions[action]
## Taskchains needs to be treated special to set the memory to all tasks
if 'TaskChain' in initial:
it = 1
while True:
t = 'Task%d'%it
it += 1
if t in initial:
payload[t] = copy.deepcopy(initial[t])
payload[t]['Memory'] = actions[action]
else:
break
if action.startswith('split'):
split_alert = (initial['RequestType'] in ['MonteCarlo'] )
for key in initial:
if key == 'SplittingAlgo' and (initial[key] in ['EventBased']):
split_alert = True
elif key.startswith('Task') and key != 'TaskChain':
for key2 in initial[key]:
if key2 == 'TaskName':
print "task",task.split('/')[-1]
print "TaskName",initial[key][key2]
if (initial[key][key2] == task) and (initial[key][key2] in ['EventBased']):
split_alert = True
if split_alert:
sendLog('actor','Cannot change splitting for %s'%initial['RequestName'],level='warning')
print "I should not be doing splitting for this type of request",initial['RequestName']
return None
acdc_round = 0
initial_string = payload['RequestString']
if initial_string.startswith('ACDC'):
if initial_string[4].isdigit():
acdc_round = int(initial_string[4])
acdc_round += 1
initial_string = initial_string.replace('ACDC_','').replace('ACDC%d_'%(acdc_round-1),'')
payload['RequestString'] = 'ACDC%d_%s'%(acdc_round,initial_string)
payload['InitialTaskPath'] = task
if not do:
print json.dumps( payload, indent=2)
return None
print "ACDC payload"
# print json.dumps( payload , indent=2)
print actions
## submit here
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error in making ACDC for",initial["RequestName"]
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error twice in making ACDC for",initial["RequestName"]
sendLog('actor','Failed twice in making ACDCs for %s!'%initial['RequestName'],level='critical')
return None
## change splitting if requested
if actions:
for action in actions:
if action.startswith('split'):
acdcInfo = workflowInfo(url, acdc)
splittings = acdcInfo.getSplittings()
if actions[action] != 'Same' and actions[action] != 'max':
factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split:
print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
split[act] /= factor
print "to",split[act]
break
split['requestName'] = acdc
print "changing the splitting of",acdc
print json.dumps( split, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, acdc, split )
elif 'max' in actions[action]:
for split in splittings:
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split:
print "Changing %s (%d) "%( act, split[act]),
split[act] = 1
print "to max splitting ",split[act]
break
split['requestName'] = acdc
print "changing the splitting of",acdc
print json.dumps( split, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, acdc, split )
data = reqMgrClient.setWorkflowApproved(url, acdc)
print data
return acdc
def singleRecovery(url, task, initial, actions, do=False):
payload = {
"Requestor": os.getenv('USER'),
"Group": 'DATAOPS',
"RequestType": "Resubmission",
"ACDCServer": initial['CouchURL'],
"ACDCDatabase": "acdcserver",
"OriginalRequestName": initial['RequestName']
}
copy_over = [
'PrepID', 'RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group',
'Memory', 'RequestString', 'CMSSWVersion'
]
for c in copy_over:
payload[c] = copy.deepcopy(initial[c])
#a massage ? boost the recovery over the initial wf
payload['RequestPriority'] *= 10
if actions:
for action in actions:
#if action.startswith('split'):
# factor = int(action.split('-')[-1]) if '-' in action else 2
# print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
# ## mention it's taking 2 times longer to have a 2 times finer splitting
# payload['TimePerEvent'] = factor*payload['TimePerEvent']
if action.startswith('mem'):
increase = int(
action.split('-')[-1]) if '-' in action else 1000
## increase the memory requirement by 1G
payload['Memory'] += increase
if action.startswith('split') and (
initial['RequestType'] in ['MonteCarlo'] or
(initial['RequestType'] in ['TaskChain']
and not 'InputDataset' in initial['Task1'])):
print "I should not be doing splitting for this type of request", initial[
'RequestName']
return None
if payload['RequestString'].startswith('ACDC'):
print "This is not allowed yet"
return None
payload['RequestString'] = 'ACDC_' + payload['RequestString']
payload['InitialTaskPath'] = task
if not do:
print json.dumps(payload, indent=2)
return None
print json.dumps(payload, indent=2)
## submit
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error in making ACDC for", initial["RequestName"]
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error twice in making ACDC for", initial["RequestName"]
return None
## perform modifications
if actions:
for action in actions:
if action.startswith('split'):
factor = int(action.split('-')[-1]) if '-' in action else 2
acdcInfo = workflowInfo(url, acdc)
splittings = acdcInfo.getSplittings()
for split in splittings:
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split:
print "Changing %s (%d) by a factor %d" % (
act, split[act], factor),
split[act] /= factor
print "to", split[act]
break
split['requestName'] = acdc
print "changing the splitting of", acdc
print json.dumps(split, indent=2)
print reqMgrClient.setWorkflowSplitting(url, acdc, split)
data = reqMgrClient.setWorkflowApproved(url, acdc)
print data
return acdc
def spawn_harvesting(url, wfi, sites_for_DQMHarvest):
SI = global_SI()
all_OK = {}
requests = []
outputs = wfi.request['OutputDatasets']
if ('EnableHarvesting' in wfi.request
and not wfi.request['EnableHarvesting']) and (
'DQMConfigCacheID' in wfi.request
and wfi.request['DQMConfigCacheID']):
if not 'MergedLFNBase' in wfi.request:
print "f****d up"
sendEmail('screwed up wl cache',
'%s wl cache is bad' % (wfi.request['RequestName']))
all_OK['fake'] = False
return all_OK, requests
wfi = workflowInfo(url, wfi.request['RequestName'])
dqms = [out for out in outputs if '/DQM' in out]
#if not all([in_full[dqm_input] for dqm_input in dqms]):
# wfi.sendLog('closor',"will not be able to assign the harvesting: holding up")
# for dqm_input in dqms:
# all_OK[dqm_input] = False
## raise the subscription to high priority
for dqm_input in dqms:
## handle it properly
harvesting_schema = {
'Requestor': os.getenv('USER'),
'RequestType': 'DQMHarvest',
'Group': 'DATAOPS'
}
copy_over = [
'AcquisitionEra',
'ProcessingString',
'DQMUploadUrl',
'CMSSWVersion',
'CouchDBName',
'CouchWorkloadDBName',
'ConfigCacheUrl',
'DbsUrl',
'inputMode',
'DQMConfigCacheID',
'OpenRunningTimeout',
'ScramArch',
'CMSSWVersion',
'Campaign',
'Memory', #dummy
'SizePerEvent', #dummy
'GlobalTag', #dummy
]
for item in copy_over:
if item in wfi.request:
harvesting_schema[item] = copy.deepcopy(wfi.request[item])
else:
print item, "is not in initial schema"
harvesting_schema['InputDataset'] = dqm_input
harvesting_schema['TimePerEvent'] = 1
harvesting_schema['PrepID'] = 'Harvest-' + wfi.request['PrepID']
if len(wfi.request['RequestString']) > 60:
wfi.request['RequestString'] = wfi.request[
'RequestString'][:60]
print "truncating request string", wfi.request['RequestString']
harvesting_schema[
'RequestString'] = 'HARVEST-' + wfi.request['RequestString']
harvesting_schema['DQMHarvestUnit'] = 'byRun'
harvesting_schema['RequestPriority'] = min(
wfi.request['RequestPriority'] * 10, 999999)
harvest_request = reqMgrClient.submitWorkflow(
url, harvesting_schema)
if not harvest_request:
print "Error in making harvesting for", wfi.request[
'RequestName']
print "schema"
print json.dumps(harvesting_schema, indent=2)
harvest_request = reqMgrClient.submitWorkflow(
url, harvesting_schema)
if not harvest_request:
print "Error twice in harvesting for", wfi.request[
'RequestName']
print "schema"
print json.dumps(harvesting_schema, indent=2)
if harvest_request:
requests.append(harvest_request)
## should we protect for setting approved ? no, it's notified below, assignment will fail, likely
data = reqMgrClient.setWorkflowApproved(url, harvest_request)
print "created", harvest_request, "for harvesting of", dqm_input
wfi.sendLog(
'closor', "created %s for harvesting of %s" %
(harvest_request, dqm_input))
## assign it directly
team = wfi.request['Team']
parameters = {
'SiteWhitelist': [
SI.SE_to_CE(se)
for se in wfi.request['NonCustodialSites']
],
'AcquisitionEra':
wfi.acquisitionEra(),
'ProcessingString':
wfi.processingString(),
'MergedLFNBase':
wfi.request['MergedLFNBase'],
'ProcessingVersion':
wfi.request['ProcessingVersion'],
'execute':
True
}
#if in_full[dqm_input]:
# print "using full copy at",in_full[dqm_input]
# parameters['SiteWhitelist'] = [SI.SE_to_CE(se) for se in in_full[dqm_input]]
#else:
# print "cannot do anything if not having a full copy somewhere"
# all_OK[dqm_input]=False
# continue
parameters['SiteWhitelist'] = sites_for_DQMHarvest
result = reqMgrClient.assignWorkflow(url, harvest_request,
team, parameters)
if not result:
#sendEmail('harvesting request created','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
wfi.sendLog(
'closor',
'%s was created at announcement of %s in %s, failed to assign'
% (harvest_request, dqm_input,
wfi.request['RequestName']))
sendLog(
'closor',
'%s was created at announcement of %s in %s, failed to assign'
% (harvest_request, dqm_input,
wfi.request['RequestName']),
level='critical')
else:
#sendEmail('harvesting request assigned','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
wfi.sendLog(
'closor',
'%s was created at announcement of %s in %s, and assigned'
% (harvest_request, dqm_input,
wfi.request['RequestName']))
else:
#print "could not make the harvesting for",wfo.name,"not announcing"
wfi.sendLog('closor', "could not make the harvesting request")
sendLog('closor',
"could not make the harvesting request for %s" %
wfi.request['RequestName'],
level='critical')
all_OK[dqm_input] = False
return (all_OK, requests)
def singleRecovery(url, task , initial, actions, do=False):
payload = {
"Requestor" : os.getenv('USER'),
"Group" : 'DATAOPS',
"RequestType" : "Resubmission",
"ACDCServer" : initial['CouchURL'],
"ACDCDatabase" : "acdcserver",
"OriginalRequestName" : initial['RequestName']
}
copy_over = ['PrepID','RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ,'CMSSWVersion']
for c in copy_over:
payload[c] = copy.deepcopy(initial[c])
#a massage ? boost the recovery over the initial wf
payload['RequestPriority'] *= 10
if actions:
for action in actions:
#if action.startswith('split'):
# factor = int(action.split('-')[-1]) if '-' in action else 2
# print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
# ## mention it's taking 2 times longer to have a 2 times finer splitting
# payload['TimePerEvent'] = factor*payload['TimePerEvent']
if action.startswith('mem'):
increase = int(action.split('-')[-1]) if '-' in action else 1000
## increase the memory requirement by 1G
payload['Memory'] += increase
if action.startswith('split') and (initial['RequestType'] in ['MonteCarlo'] or (initial['RequestType'] in ['TaskChain'] and not 'InputDataset' in initial['Task1'])):
print "I should not be doing splitting for this type of request",initial['RequestName']
return None
if payload['RequestString'].startswith('ACDC'):
print "This is not allowed yet"
return None
payload['RequestString'] = 'ACDC_'+payload['RequestString']
payload['InitialTaskPath'] = task
if not do:
print json.dumps( payload, indent=2)
return None
print json.dumps( payload , indent=2)
## submit
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error in making ACDC for",initial["RequestName"]
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error twice in making ACDC for",initial["RequestName"]
return None
## perform modifications
if actions:
for action in actions:
if action.startswith('split'):
factor = int(action.split('-')[-1]) if '-' in action else 2
acdcInfo = workflowInfo(url, acdc)
splittings = acdcInfo.getSplittings()
for split in splittings:
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split:
print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
split[act] /= factor
print "to",split[act]
break
split['requestName'] = acdc
print "changing the splitting of",acdc
print json.dumps( split, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, acdc, split )
data = reqMgrClient.setWorkflowApproved(url, acdc)
print data
return acdc
def spawn_harvesting(url, wfi , in_full):
SI = siteInfo()
all_OK = {}
requests = []
outputs = wfi.request['OutputDatasets']
if ('EnableHarvesting' in wfi.request and wfi.request['EnableHarvesting']) or ('DQMConfigCacheID' in wfi.request and wfi.request['DQMConfigCacheID']):
if not 'MergedLFNBase' in wfi.request:
print "f****d up"
sendEmail('screwed up wl cache','%s wl cache is bad'%(wfi.request['RequestName']))
all_OK['fake'] = False
return all_OK,requests
wfi = workflowInfo(url, wfi.request['RequestName'])
dqms = [out for out in outputs if '/DQM' in out]
if not all([in_full[dqm_input] for dqm_input in dqms]):
wfi.sendLog('closor',"will not be able to assign the harvesting: holding up")
for dqm_input in dqms:
all_OK[dqm_input] = False
return all_OK,requests
for dqm_input in dqms:
## handle it properly
harvesting_schema = {
'Requestor': os.getenv('USER'),
'RequestType' : 'DQMHarvest',
'Group' : 'DATAOPS'
}
copy_over = ['ProcessingString',
'DQMUploadUrl',
'CMSSWVersion',
'CouchDBName',
'CouchWorkloadDBName',
'CouchURL',
'DbsUrl',
'inputMode',
'DQMConfigCacheID',
'OpenRunningTimeout',
'ScramArch',
'CMSSWVersion',
'Campaign',
'Memory', #dummy
'SizePerEvent', #dummy
'GlobalTag', #dummy
]
for item in copy_over:
harvesting_schema[item] = copy.deepcopy(wfi.request[item])
harvesting_schema['InputDataset'] = dqm_input
harvesting_schema['TimePerEvent'] = 1
harvesting_schema['PrepID'] = 'Harvest-'+wfi.request['PrepID']
harvesting_schema['RequestString'] = 'HARVEST-'+wfi.request['RequestString']
harvesting_schema['DQMHarvestUnit'] = 'byRun'
harvesting_schema['ConfigCacheUrl'] = harvesting_schema['CouchURL'] ## uhm, how stupid is that ?
harvesting_schema['RequestPriority'] = wfi.request['RequestPriority']*10
harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema)
if not harvest_request:
print "Error in making harvesting for",wfo.name
print "schema"
print json.dumps( harvesting_schema, indent = 2)
harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema)
if not harvest_request:
print "Error twice in harvesting for",wfo.name
print "schema"
print json.dumps( harvesting_schema, indent = 2)
if harvest_request:
requests.append( harvest_request )
## should we protect for setting approved ? no, it's notified below, assignment will fail, likely
data = reqMgrClient.setWorkflowApproved(url, harvest_request)
print "created",harvest_request,"for harvesting of",dqm_input
wfi.sendLog('closor',"created %s for harvesting of %s"%( harvest_request, dqm_input))
## assign it directly
team = wfi.request['Teams'][0]
parameters={
'SiteWhitelist' : [SI.SE_to_CE(se) for se in wfi.request['NonCustodialSites']],
'AcquisitionEra' : wfi.acquisitionEra(),
'ProcessingString' : wfi.processingString(),
'MergedLFNBase' : wfi.request['MergedLFNBase'],
'ProcessingVersion' : wfi.request['ProcessingVersion'],
'execute' : True
}
if in_full[dqm_input]:
print "using full copy at",in_full[dqm_input]
parameters['SiteWhitelist'] = [SI.SE_to_CE(se) for se in in_full[dqm_input]]
else:
print "cannot do anything if not having a full copy somewhere"
all_OK[dqm_input]=False
continue
result = reqMgrClient.assignWorkflow(url, harvest_request, team, parameters)
if not result:
sendEmail('harvesting request created','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
else:
sendEmail('harvesting request assigned','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
wfi.sendLog('closor','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']))
else:
print "could not make the harvesting for",wfo.name,"not announcing"
wfi.sendLog('closor',"could not make the harvesting request")
all_OK[dqm_input]=False
return (all_OK, requests)
def singleRecovery(url, task, initial, actions, do=False):
print "Inside single recovery!"
payload = {
"Requestor": os.getenv('USER'),
"Group": 'DATAOPS',
"RequestType": "Resubmission",
"ACDCServer": initial['ConfigCacheUrl'],
"ACDCDatabase": "acdcserver",
"OriginalRequestName": initial['RequestName'],
"OpenRunningTimeout": 0
}
copy_over = [
'PrepID', 'Campaign', 'RequestPriority', 'TimePerEvent',
'SizePerEvent', 'Group', 'Memory', 'RequestString', 'CMSSWVersion'
]
for c in copy_over:
if c in initial:
payload[c] = copy.deepcopy(initial[c])
else:
print c, "not in the initial payload"
#a massage ? boost the recovery over the initial wf
# payload['RequestPriority'] *= 10
#Max priority is 1M
payload['RequestPriority'] = min(500000, payload['RequestPriority'] *
2) ## never above 500k
#change parameters based on actions here
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[
action] != 'Same':
#if multicore parameter is also used, need to scale memory by the new number of cores
if 'multicore' in actions and actions['multicore'] != "":
continue
## Taskchains needs to be treated special to set the memory to all tasks
set_to = int(actions[action])
if 'TaskChain' in initial:
mem_dict = {}
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in initial:
tname = payload.setdefault(t,
initial[t])['TaskName']
mem = mem_dict.setdefault(tname,
payload[t]['Memory'])
mem_dict[tname] = set_to
else:
break
payload['Memory'] = mem_dict
print "Memory set to: ", json.dumps(mem_dict, indent=2)
else:
payload['Memory'] = set_to
print "Memory set to: ", set_to
if action.startswith('multicore') and actions[action] != "":
set_to = int(actions[action])
## Taskchains needs to be treated special to set the multicore and memory values to all tasks
if 'TaskChain' in initial:
mem_dict = payload['Memory'] if type(
payload['Memory']) == dict else {}
core_dict = {}
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in initial:
tname = payload.setdefault(t,
initial[t])['TaskName']
mem = mem_dict.setdefault(tname,
payload[t]['Memory'])
#Need to scale the memory by the new number of cores
initial_cores = payload[t].setdefault(
'Multicore', 1)
if 'memory' in actions and actions[
'memory'] != "" and actions[
'memory'] != 'Same':
mem = actions['memory']
fraction_constant = 0.4
mem_per_core_c = int((1 - fraction_constant) *
mem / float(initial_cores))
mem_dict[tname] = int(mem +
(set_to - initial_cores) *
mem_per_core_c)
core_dict[tname] = set_to
print "For ", t
print "Multicore set to ", set_to
print "Memory set to ", mem_dict[tname]
else:
break
payload['Memory'] = mem_dict
payload['Multicore'] = core_dict
else:
#Need to scale the memory by the new number of cores
initial_cores = initial.setdefault('Multicore', 1)
mem = payload['Memory']
if 'memory' in actions and actions[
'memory'] != "" and actions['memory'] != 'Same':
mem = actions['memory']
fraction_constant = 0.4
mem_per_core_c = int(
(1 - fraction_constant) * mem / float(initial_cores))
payload['Multicore'] = set_to
payload['Memory'] = int(mem + (set_to - initial_cores) *
mem_per_core_c)
print "Multicore set to ", set_to
print "Memory set to ", payload['Memory']
if action.startswith('split'):
split_alert = (initial['RequestType'] in ['MonteCarlo'])
for key in initial:
if key == 'SplittingAlgo' and (initial[key]
in ['EventBased']):
split_alert = True
elif key.startswith('Task') and key != 'TaskChain':
for key2 in initial[key]:
if key2 == 'TaskName':
this_taskname = initial[key][key2]
recover_task = task.split('/')[-1]
print "For recovery of task", recover_task
print "Looking at task", this_taskname
if (recover_task == this_taskname) and (
initial[key]['SplittingAlgo']
in ['EventBased']):
## the task to be recovered is actually of the wrong type to allow change of splitting
sendLog(
'actor',
'To recover on %s, changing the splitting on %s is not really allowed and this will be ignored instead of failing acdc.'
%
(task, initial[key]['SplittingAlgo']),
level='critical')
## do not send an alert and stop the acdc
#split_alert = True
if split_alert:
sendLog('actor',
'Cannot change splitting for %s' %
initial['RequestName'],
level='critical')
print "I should not be doing splitting for this type of request", initial[
'RequestName']
return None
acdc_round = 0
initial_string = payload['RequestString']
if initial_string.startswith('ACDC'):
if initial_string[4].isdigit():
acdc_round = int(initial_string[4])
acdc_round += 1
initial_string = initial_string.replace('ACDC_', '').replace(
'ACDC%d_' % (acdc_round - 1), '')
payload['RequestString'] = 'ACDC%d_%s' % (acdc_round, initial_string)
payload['InitialTaskPath'] = task
if not do:
print json.dumps(payload, indent=2)
return None
print "ACDC payload"
# print json.dumps( payload , indent=2)
print actions
## submit here
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error in making ACDC for", initial["RequestName"]
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error twice in making ACDC for", initial["RequestName"]
sendLog('actor',
'Failed twice in making ACDCs for %s!' %
initial['RequestName'],
level='critical')
return None
## change splitting if requested
if actions:
for action in actions:
if action.startswith('split'):
acdcInfo = workflowInfo(url, acdc)
splittings = acdcInfo.getSplittingsNew(strip=True)
if actions[action] != 'Same' and actions[action] != 'max':
factor = int(
actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
split_par = split['splitParams']
if split['splitAlgo'] in ['EventBased']:
sendLog(
'actor',
"Changing the splitting on %s for %s is not permitted. Not changing."
% (split['splitAlgo'], initial["RequestName"]),
level='critical')
continue
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split_par:
print "Changing %s (%d) by a factor %d" % (
act, split_par[act], factor),
split_par[act] /= factor
print "to", split_par[act]
break
#split['requestName'] = acdc
#print "changing the splitting of",acdc
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, acdc, split )
elif 'max' in actions[action]:
for split in splittings:
split_par = split['splitParams']
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split_par:
print "Changing %s (%d) " % (act,
split_par[act]),
split_par[act] = 1
print "to max splitting ", split_par[act]
break
#split['requestName'] = acdc
#print "changing the splitting of",acdc
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, acdc, split )
print "changing the splitting of", acdc
print json.dumps(splittings, indent=2)
done = reqMgrClient.setWorkflowSplitting(url, acdc, splittings)
## check on done == True
data = reqMgrClient.setWorkflowApproved(url, acdc)
print data
return acdc
def singleClone(url, wfname, actions, comment, do=False):
wfi = workflowInfo(url, wfname)
payload = wfi.getSchema()
initial = wfi.request
payload['Requestor'] = os.getenv('USER')
payload['Group'] = 'DATAOPS'
payload['OriginalRequestName'] = initial['RequestName']
payload['RequestPriority'] = initial['RequestPriority']
if 'ProcessingVersion' in initial:
payload['ProcessingVersion'] = int(initial['ProcessingVersion']) +1
else:
payload['ProcessingVersion'] = 2
payload = reqMgrClient.purgeClonedSchema( payload )
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
if 'TaskChain' in payload:
print "Setting memory for clone of task chain"
mem_dict = {}
it=1
while True:
t = 'Task%d'%it
it+=1
if t in payload:
tname = payload[t]['TaskName']
mem_dict[tname] = int(actions[action])
print "Memory set for Task%d"%it
else:
break
payload['Memory'] = mem_dict
else:
print "Setting memory for non-taskchain workflow"
payload['Memory'] = int(actions[action])
print "Memory set to " + actions[action]
print "Clone payload"
# print json.dumps( payload , indent=2)
print actions
#Create clone
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error in making clone for",initial["RequestName"]
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error twice in making clone for",initial["RequestName"]
sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"],level='critical')
wfi.sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"])
return None
if actions:
for action in actions:
if action.startswith('split'):
cloneinfo = workflowInfo(url, clone)
splittings = cloneinfo.getSplittingsNew(strip=True)
if actions[action] != 'Same' and actions[action] != 'max' and actions[action] != '':
factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
split_par = split['splitParams']
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split_par:
wfi.sendLog('actor','Changing %s (%d) by a factor %d'%( act, split_par[act], factor))
split_par[act] /= factor
print "to",split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
elif 'max' in actions[action]:
for split in splittings:
split_par = split['splitParams']
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split_par:
wfi.sendLog('actor','Max splitting set for %s (%d'%( act, split_par[act]))
print "Changing %s (%d) "%( act, split_par[act]),
split_par[act] = 1
print "to max splitting ",split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
print "changing the splitting of",clone
print json.dumps( splittings, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, clone, splittings )
#Approve
data = reqMgrClient.setWorkflowApproved(url, clone)
#wfi.sendLog('actor','Cloned into %s'%clone)
# wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
# wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)
print data
return clone
def singleClone(url, wfname, actions, comment, do=False):
wfi = workflowInfo(url, wfname)
payload = wfi.getSchema()
initial = wfi.request
payload['Requestor'] = os.getenv('USER')
payload['Group'] = 'DATAOPS'
payload['OriginalRequestName'] = initial['RequestName']
payload['RequestPriority'] = initial['RequestPriority']
if 'ProcessingVersion' in initial:
payload['ProcessingVersion'] = int(initial['ProcessingVersion']) + 1
else:
payload['ProcessingVersion'] = 2
## drop parameters on the way to reqmgr2
paramBlacklist = [
'BlockCloseMaxEvents', 'BlockCloseMaxFiles', 'BlockCloseMaxSize',
'BlockCloseMaxWaitTime', 'CouchWorkloadDBName', 'CustodialGroup',
'CustodialSubType', 'Dashboard', 'GracePeriod', 'HardTimeout',
'InitialPriority', 'inputMode', 'MaxMergeEvents', 'MaxMergeSize',
'MaxRSS', 'MaxVSize', 'MinMergeSize', 'NonCustodialGroup',
'NonCustodialSubType', 'OutputDatasets', 'ReqMgr2Only', 'RequestDate'
'RequestorDN', 'RequestName', 'RequestStatus', 'RequestTransition',
'RequestWorkflow', 'SiteWhitelist', 'SoftTimeout', 'SoftwareVersions',
'SubscriptionPriority', 'Team', 'timeStamp', 'TrustSitelists',
'TrustPUSitelists', 'TotalEstimatedJobs', 'TotalInputEvents',
'TotalInputLumis', 'TotalInputFiles', 'checkbox', 'DN',
'AutoApproveSubscriptionSites', 'NonCustodialSites', 'CustodialSites',
'OriginalRequestName', 'Teams', 'OutputModulesLFNBases',
'SiteBlacklist', 'AllowOpportunistic', '_id', 'Override'
]
for p in paramBlacklist:
if p in payload:
payload.pop(p)
taskParamBlacklist = ['EventsPerJob']
for i in range(1, 100):
t = 'Task%s' % i
if not t in payload: break
for p in taskParamBlacklist:
if p in payload[t]:
payload[t].pop(p)
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[
action] != 'Same':
if 'TaskChain' in payload:
print "Setting memory for clone of task chain"
mem_dict = {}
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in payload:
tname = payload[t]['TaskName']
mem_dict[tname] = int(actions[action])
print "Memory set for Task%d" % it
else:
break
payload['Memory'] = mem_dict
else:
print "Setting memory for non-taskchain workflow"
payload['Memory'] = int(actions[action])
print "Memory set to " + actions[action]
print "Clone payload"
# print json.dumps( payload , indent=2)
print actions
#Create clone
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error in making clone for", initial["RequestName"]
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error twice in making clone for", initial["RequestName"]
sendLog('actor',
'Failed to make a clone twice for %s!' %
initial["RequestName"],
level='critical')
wfi.sendLog(
'actor', 'Failed to make a clone twice for %s!' %
initial["RequestName"])
return None
if actions:
for action in actions:
if action.startswith('split'):
cloneinfo = workflowInfo(url, clone)
splittings = cloneinfo.getSplittingsNew(strip=True)
if actions[action] != 'Same' and actions[
action] != 'max' and actions[action] != '':
factor = int(
actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
split_par = split['splitParams']
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split_par:
wfi.sendLog(
'actor',
'Changing %s (%d) by a factor %d' %
(act, split_par[act], factor))
split_par[act] /= factor
print "to", split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
elif 'max' in actions[action]:
for split in splittings:
split_par = split['splitParams']
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split_par:
wfi.sendLog(
'actor', 'Max splitting set for %s (%d' %
(act, split_par[act]))
print "Changing %s (%d) " % (act,
split_par[act]),
split_par[act] = 1
print "to max splitting ", split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
print "changing the splitting of", clone
print json.dumps(splittings, indent=2)
print reqMgrClient.setWorkflowSplitting(url, clone, splittings)
#Approve
data = reqMgrClient.setWorkflowApproved(url, clone)
wfi.sendLog('actor', 'Cloned into %s' % clone)
# wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
# wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)
print data
return clone
def singleRecovery(url, task , initial, actions, do=False):
payload = {
"Requestor" : os.getenv('USER'),
"Group" : 'DATAOPS',
"RequestType" : "Resubmission",
"ACDCServer" : initial['ConfigCacheUrl'],
"ACDCDatabase" : "acdcserver",
"OriginalRequestName" : initial['RequestName'],
"OpenRunningTimeout" : 0
}
copy_over = ['PrepID', 'Campaign', 'RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ,'CMSSWVersion']
for c in copy_over:
if c in initial:
payload[c] = copy.deepcopy(initial[c])
else:
print c,"not in the initial payload"
#a massage ? boost the recovery over the initial wf
payload['RequestPriority'] *= 2
payload['RequestPriority'] = min(500000, payload['RequestPriority'])
if actions:
for action in actions:
#if action.startswith('split'):
# factor = int(action.split('-')[-1]) if '-' in action else 2
# print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
# ## mention it's taking 2 times longer to have a 2 times finer splitting
# payload['TimePerEvent'] = factor*payload['TimePerEvent']
if action.startswith('mem'):
arg = action.split('-',1)[-1]
increase = set_to = None
tasks,set_to = arg.split(':') if ':' in arg else (None,arg)
tasks = tasks.split(',') if tasks else []
if set_to.startswith('+'):
increase = int(set_to[1:])
else:
set_to = int(set_to)
## increase the memory requirement by 1G
if 'TaskChain' in initial:
mem_dict = {}
it = 1
while True:
t = 'Task%d'%it
it += 1
if t in initial:
tname = payload.setdefault(t, initial[t])['TaskName']
mem = mem_dict.setdefault( tname, payload[t]['Memory'])
if tasks and not tname in tasks:
print tname,"not concerned"
continue
if set_to:
mem_dict[tname] = set_to
else:
mem_dict[tname] += increase
else:
break
payload['Memory'] = mem_dict
else:
payload['Memory'] = set_to
#increase = int(action.split('-')[-1]) if '-' in action else 1000
## increase the memory requirement by 1G
#payload['Memory'] += increase
if action.startswith('split') and (initial['RequestType'] in ['MonteCarlo'] or (initial['RequestType'] in ['TaskChain'] and not 'InputDataset' in initial['Task1'])):
print "I should not be doing splitting for this type of request",initial['RequestName']
return None
if action.startswith('core'):
arg = action.split('-',1)[-1]
tasks,set_to = arg.split(':') if ':' in arg else (None,arg)
tasks = tasks.split(',') if tasks else []
set_to = int(set_to)
if 'TaskChain' in initial:
core_dict = {}
mem_dict = payload['Memory'] if type(payload['Memory'])==dict else {}
it = 1
while True:
t = 'Task%d'%it
it += 1
if t in initial:
tname = payload.setdefault(t, initial[t])['TaskName']
mcore = core_dict.setdefault(tname, payload[t]['Multicore'])
mem = mem_dict.setdefault(tname, payload[t]['Memory'])
if tasks and not tname in tasks:
print tname,"not concerned"
continue
factor = (set_to / float(mcore))
fraction_constant = 0.4
mem_per_core_c = int((1-fraction_constant) * mem / float(mcore))
##scale the memory
mem_dict[tname] += (set_to-mcore)*mem_per_core_c
## scale time/event
time_dict[tname] = payload[t]['TimePerEvent'] /factor
## set the number of cores
core_dict[tname] = set_to
else:
break
payload['Multicore'] = core_dict
##payload['TimePerEvent'] = time_dict ## cannot be used yet
else:
payload['Multicore'] = increase
acdc_round = 0
initial_string = payload['RequestString']
if initial_string.startswith('ACDC'):
if initial_string[4].isdigit():
acdc_round = int(initial_string[4])
acdc_round += 1
#print acdc_round
#print "This is not allowed yet"
#return None
initial_string = initial_string.replace('ACDC_','').replace('ACDC%d_'%(acdc_round-1),'')
payload['RequestString'] = 'ACDC%d_%s'%(acdc_round,initial_string)
payload['InitialTaskPath'] = task
if not do:
print json.dumps( payload, indent=2)
return None
print "ACDC payload"
print json.dumps( payload , indent=2)
print actions
## submit
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error in making ACDC for",initial["RequestName"]
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error twice in making ACDC for",initial["RequestName"]
return None
## perform modifications
if actions:
for action in actions:
if action.startswith('split'):
factor = int(action.split('-')[-1]) if '-' in action else 2
acdcInfo = workflowInfo(url, acdc)
splittings = acdcInfo.getSplittings()
for split in splittings:
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split:
print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
split[act] /= factor
print "to",split[act]
break
split['requestName'] = acdc
print "changing the splitting of",acdc
print json.dumps( split, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, acdc, split )
data = reqMgrClient.setWorkflowApproved(url, acdc)
print data
return acdc
def testSetWorkflowApproved(self):
# setWorkflowApproved
te = reqMgr.setWorkflowApproved(url, testwf)
if te is None:
self.fail("None output")
def rejector(url, specific, options=None):
up = componentInfo(soft=['wtc','jira'])
if not up.check(): return
if specific and specific.startswith('/'):
## this is for a dataset
print setDatasetStatus(specific, 'INVALID')
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend( session.query(Workflow).filter(Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(Workflow.name.contains(specific)).all()
if not wfs:
batches = batchInfo().content()
for bname in batches:
if specific == bname:
for pid in batches[bname]:
b_wfs = getWorkflowById(url, pid)
for wf in b_wfs:
wfs.append( session.query(Workflow).filter(Workflow.name == wf).first())
break
else:
wfs = session.query(Workflow).filter(Workflow.status == 'assistance-clone').all()
#wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
## be careful then on clone case by case
options.clone = True
print "not supposed to function yet"
return
print len(wfs),"to reject"
if len(wfs)>1:
print "\n".join( [wfo.name for wfo in wfs] )
answer = raw_input('Reject these')
if not answer.lower() in ['y','yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject",spec
return
wfi = workflowInfo(url, wfo.name)
comment=""
if options.comments: comment = ", reason: "+options.comments
if options.keep:
wfi.sendLog('rejector','invalidating the workflow by unified operator%s'%comment)
else:
wfi.sendLog('rejector','invalidating the workflow and outputs by unified operator%s'%comment)
results = invalidate(url, wfi, only_resub=True, with_output= (not options.keep))
if all(results):
print wfo.name,"rejected"
if options and options.clone:
wfo.status = 'trouble'
session.commit()
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(schema['ProcessingVersion'])+1 ## dubious str->int conversion
else:
schema['ProcessingVersion']=2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
if schema['RequestType'] == 'TaskChain':
it=1
while True:
t = 'Task%d'%it
it+=1
if t in schema:
schema[t]['Memory'] = options.Memory
else:
break
else:
schema['Memory'] = options.Memory
if options.Multicore:
## to do : set it properly in taskchains
if schema['RequestType'] == 'TaskChain':
tasks,set_to = options.Multicore.split(':') if ':' in options.Multicore else ("",options.Multicore)
set_to = int(set_to)
tasks = tasks.split(',') if tasks else ['Task1']
it = 1
while True:
tt = 'Task%d'% it
it+=1
if tt in schema:
tname = schema[tt]['TaskName']
if tname in tasks or tt in tasks:
mem = schema[tt]['Memory']
mcore = schema[tt].get('Multicore',1)
factor = (set_to / float(mcore))
fraction_constant = 0.4
mem_per_core_c = int((1-fraction_constant) * mem / float(mcore))
print "mem per core", mem_per_core_c
print "base mem", mem
## adjusting the parameter in the clone
schema[tt]['Memory'] += (set_to-mcore)*mem_per_core_c
schema[tt]['Multicore'] = set_to
schema[tt]['TimePerEvent'] /= factor
else:
break
else:
schema['Multicore'] = options.Multicore
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int,options.runs.split(','))
if options.PrepID:
schema['PrepID'] =options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1,ntask-1):
schema['Task%d'%it]['KeepOutput'] = False
schema['TaskChain'] = ntask-1
schema.pop('Task%d'%ntask)
if options.priority:
schema['RequestPriority'] = options.priority
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
schema = reqMgrClient.purgeClonedSchema( schema )
print "submitting"
if (options.to_stepchain and (schema['RequestType']=='TaskChain')):
## transform the schema into StepChain schema
print "Transforming a TaskChain into a StepChain"
mcore = 0
mem = 0
schema['RequestType'] = 'StepChain'
schema['StepChain'] = schema.pop('TaskChain')
schema['SizePerEvent'] = 0
schema['TimePerEvent'] = 0
step=1
s_n = {}
while True:
if 'Task%d'%step in schema:
sname = 'Step%d'%step
schema[sname] = schema.pop('Task%d'%step)
tmcore = schema[sname].pop('Multicore')
tmem = schema[sname].pop('Memory')
if mcore and tmcore != mcore:
wfi.sendLog('rejector','the conversion to stepchain encoutered different value of Multicore %d != %d'%( tmcore, mcore))
sendLog('rejector','the conversion of %s to stepchain encoutered different value of Multicore %d != %d'%( wfo.name, tmcore, mcore), level='critical')
mcore = max(mcore, tmcore)
mem = max(mem, tmem)
schema[sname]['StepName'] = schema[sname].pop('TaskName')
s_n[ schema[sname]['StepName'] ] = sname
if 'InputTask' in schema[sname]:
schema[sname]['InputStep'] = schema[sname].pop('InputTask')
eff = 1.
up_s = sname
while True:
## climb up a step. supposedely already all converted
up_s = s_n.get(schema[up_s].get('InputStep',None),None)
if up_s:
## multiply with the efficiency
eff *= schema[up_s].get('FilterEfficiency',1.)
else:
## or stop there
break
if not 'KeepOutput' in schema[sname]:
## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
schema[sname]['KeepOutput'] = False
schema['TimePerEvent'] += eff*schema[sname].pop('TimePerEvent')
schema['SizePerEvent'] += eff*schema[sname].pop('SizePerEvent')
step+=1
else:
break
schema['Multicore'] = mcore
schema['Memory'] = mem
print json.dumps( schema, indent=2 )
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
msg = "Error in cloning {}".format(wfo.name)
print(msg)
wfi.sendLog('rejector',msg)
# Get the error message
time.sleep(5)
data = reqMgrClient.requestManagerPost(url, "/reqmgr2/data/request", schema)
wfi.sendLog('rejector',data)
print json.dumps( schema, indent=2 )
return
print newWorkflow
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfi.sendLog('rejector','Cloned into %s by unified operator %s'%( newWorkflow, comment ))
wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False)
else:
wfo.status = 'trouble' if options.set_trouble else 'forget'
wfi.notifyRequestor('Rejected by unified operator %s'%( comment ),do_batch=False)
session.commit()
else:
msg = "Error in rejecting {}: {}".format(wfo.name,results)
print(msg)
wfi.sendLog('rejector',msg)
def testSetWorkflowApproved(self):
# setWorkflowApproved
te = reqMgr.setWorkflowApproved(url, testwf)
if te is None:
self.fail("None output")
def rejector(url, specific, options=None):
up = componentInfo(soft=['wtc'])
if not up.check(): return
if specific and specific.startswith('/'):
## this is for a dataset
print setDatasetStatus(specific, 'INVALID')
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend(
session.query(Workflow).filter(
Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(
Workflow.name.contains(specific)).all()
if not wfs:
batches = json.loads(open('batches.json').read())
for bname in batches:
if specific == bname:
for wf in batches[bname]:
wfs.append(
session.query(Workflow).filter(
Workflow.name == wf).first())
else:
wfs = session.query(Workflow).filter(
Workflow.status == 'assistance-clone').all()
#wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
## be careful then on clone case by case
options.clone = True
print "not supposed to function yet"
return
print len(wfs), "to reject"
if len(wfs) > 1:
print "\n".join([wfo.name for wfo in wfs])
answer = raw_input('Reject these')
if not answer.lower() in ['y', 'yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject", spec
return
results = []
wfi = workflowInfo(url, wfo.name)
datasets = set(wfi.request['OutputDatasets'])
reqMgrClient.invalidateWorkflow(
url, wfo.name, current_status=wfi.request['RequestStatus'])
comment = ""
if options.comments: comment = ", reason: " + options.comments
wfi.sendLog(
'rejector',
'invalidating the workflow by unified operator%s' % comment)
## need to find the whole familly and reject the whole gang
familly = getWorkflowById(url, wfi.request['PrepID'], details=True)
for fwl in familly:
if fwl['RequestDate'] < wfi.request['RequestDate']: continue
if fwl['RequestType'] != 'Resubmission': continue
## does not work on second order acd
#if 'OriginalRequestName' in fwl and fwl['OriginalRequestName'] != wfi.request['RequestName']: continue
print "rejecting", fwl['RequestName']
reqMgrClient.invalidateWorkflow(
url,
fwl['RequestName'],
current_status=fwl['RequestStatus'],
cascade=False)
datasets.update(fwl['OutputDatasets'])
for dataset in datasets:
if options.keep:
print "keeping", dataset, "in its current status"
else:
results.append(setDatasetStatus(dataset, 'INVALID'))
pass
if all(map(lambda result: result in ['None', None, True], results)):
print wfo.name, "and", datasets, "are rejected"
if options and options.clone:
wfo.status = 'trouble'
session.commit()
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(
schema['ProcessingVersion']
) + 1 ## dubious str->int conversion
else:
schema['ProcessingVersion'] = 2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
if schema['RequestType'] == 'TaskChain':
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in schema:
schema[t]['Memory'] = options.Memory
else:
break
else:
schema['Memory'] = options.Memory
if options.Multicore:
## to do : set it properly in taskchains
if schema['RequestType'] == 'TaskChain':
tasks, set_to = options.Multicore.split(
':') if ':' in options.Multicore else (
"", options.Multicore)
set_to = int(set_to)
tasks = tasks.split(',') if tasks else ['Task1']
it = 1
while True:
tt = 'Task%d' % it
it += 1
if tt in schema:
tname = schema[tt]['TaskName']
if tname in tasks or tt in tasks:
mem = schema[tt]['Memory']
mcore = schema[tt].get('Multicore', 1)
factor = (set_to / float(mcore))
fraction_constant = 0.4
mem_per_core_c = int(
(1 - fraction_constant) * mem /
float(mcore))
print "mem per core", mem_per_core_c
print "base mem", mem
## adjusting the parameter in the clone
schema[tt]['Memory'] += (
set_to - mcore) * mem_per_core_c
schema[tt]['Multicore'] = set_to
schema[tt]['TimePerEvent'] /= factor
else:
break
else:
schema['Multicore'] = options.Multicore
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int, options.runs.split(','))
if options.PrepID:
schema['PrepID'] = options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1, ntask - 1):
schema['Task%d' % it]['KeepOutput'] = False
schema['TaskChain'] = ntask - 1
schema.pop('Task%d' % ntask)
if options.priority:
schema['RequestPriority'] = options.priority
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
schema = reqMgrClient.purgeClonedSchema(schema)
print "submitting"
if (options.to_stepchain
and (schema['RequestType'] == 'TaskChain')):
## transform the schema into StepChain schema
print "Transforming a TaskChain into a StepChain"
schema['RequestType'] = 'StepChain'
schema['StepChain'] = schema.pop('TaskChain')
schema['SizePerEvent'] = 0
schema['TimePerEvent'] = 0
step = 1
s_n = {}
while True:
if 'Task%d' % step in schema:
schema['Step%d' % step] = schema.pop('Task%d' %
step)
schema['Step%d' % step]['StepName'] = schema[
'Step%d' % step].pop('TaskName')
s_n[schema['Step%d' %
step]['StepName']] = 'Step%d' % step
if 'InputTask' in schema['Step%d' % step]:
schema['Step%d' % step]['InputStep'] = schema[
'Step%d' % step].pop('InputTask')
eff = 1.
up_s = 'Step%d' % step
while True:
## climb up a step. supposedely already all converted
up_s = s_n.get(
schema[up_s].get('InputStep', None), None)
if up_s:
## multiply with the efficiency
eff *= schema[up_s].get(
'FilterEfficiency', 1.)
else:
## or stop there
break
if not 'KeepOutput' in schema['Step%d' % step]:
## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
schema['Step%d' % step]['KeepOutput'] = False
schema['TimePerEvent'] += eff * schema[
'Step%d' % step].pop('TimePerEvent')
schema['SizePerEvent'] += eff * schema[
'Step%d' % step].pop('SizePerEvent')
step += 1
else:
break
print json.dumps(schema, indent=2)
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
print "error in cloning", wfo.name
print json.dumps(schema, indent=2)
return
print newWorkflow
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfi.sendLog(
'rejector', 'Cloned into %s by unified operator %s' %
(newWorkflow, comment))
wfi.notifyRequestor('Cloned into %s by unified operator %s' %
(newWorkflow, comment),
do_batch=False)
else:
wfo.status = 'trouble' if options.set_trouble else 'forget'
wfi.notifyRequestor('Rejected by unified operator %s' %
(comment),
do_batch=False)
session.commit()
else:
print "error in rejecting", wfo.name, results
def rejector(url, specific, options=None):
up = componentInfo(soft=['wtc', 'jira'])
#if not up.check(): return
if specific and specific.startswith('/'):
## this is for a dataset
print setDatasetStatus(specific, 'INVALID')
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend(
session.query(Workflow).filter(
Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(
Workflow.name.contains(specific)).all()
if not wfs:
batches = batchInfo().content()
for bname in batches:
if specific == bname:
for pid in batches[bname]:
b_wfs = getWorkflowById(url, pid)
for wf in b_wfs:
wfs.append(
session.query(Workflow).filter(
Workflow.name == wf).first())
break
else:
wfs = session.query(Workflow).filter(
Workflow.status == 'assistance-clone').all()
#wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
## be careful then on clone case by case
options.clone = True
print "not supposed to function yet"
return
print len(wfs), "to reject"
if len(wfs) > 1:
print "\n".join([wfo.name for wfo in wfs])
answer = raw_input('Reject these')
if not answer.lower() in ['y', 'yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject", spec
return
wfi = workflowInfo(url, wfo.name)
comment = ""
if options.comments: comment = ", reason: " + options.comments
if options.keep:
wfi.sendLog(
'rejector',
'invalidating the workflow by unified operator%s' % comment)
else:
wfi.sendLog(
'rejector',
'invalidating the workflow and outputs by unified operator%s' %
comment)
results = invalidate(url,
wfi,
only_resub=True,
with_output=(not options.keep))
if all(results):
print wfo.name, "rejected"
if options and options.clone:
wfo.status = 'trouble'
session.commit()
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(
schema['ProcessingVersion']
) + 1 ## dubious str->int conversion
else:
schema['ProcessingVersion'] = 2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
if schema['RequestType'] == 'TaskChain':
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in schema:
schema[t]['Memory'] = options.Memory
else:
break
else:
schema['Memory'] = options.Memory
if options.short_task and schema['RequestType'] == 'TaskChain':
translate = {}
it = 1
while True:
tt = 'Task%d' % it
if tt in schema:
tname = schema[tt]['TaskName']
ntname = 'T%d' % it
translate[tname] = ntname
it += 1
schema[tt]['TaskName'] = ntname
if 'InputTask' in schema[tt]:
itname = schema[tt]['InputTask']
schema[tt]['InputTask'] = translate[itname]
else:
break
for k in schema.get('ProcessingString', {}).keys():
schema['ProcessingString'][
translate[k]] = schema['ProcessingString'].pop(k)
for k in schema.get('AcquisitionEra', {}).keys():
schema['AcquisitionEra'][
translate[k]] = schema['AcquisitionEra'].pop(k)
if options.Multicore:
## to do : set it properly in taskchains
if schema['RequestType'] == 'TaskChain':
tasks, set_to = options.Multicore.split(
':') if ':' in options.Multicore else (
"", options.Multicore)
set_to = int(set_to)
tasks = tasks.split(',') if tasks else ['Task1']
it = 1
while True:
tt = 'Task%d' % it
it += 1
if tt in schema:
tname = schema[tt]['TaskName']
if tname in tasks or tt in tasks:
mem = schema[tt]['Memory']
mcore = schema[tt].get('Multicore', 1)
factor = (set_to / float(mcore))
fraction_constant = 0.4
mem_per_core_c = int(
(1 - fraction_constant) * mem /
float(mcore))
print "mem per core", mem_per_core_c
print "base mem", mem
## adjusting the parameter in the clone
schema[tt]['Memory'] += (
set_to - mcore) * mem_per_core_c
schema[tt]['Multicore'] = set_to
schema[tt]['TimePerEvent'] /= factor
else:
break
else:
schema['Multicore'] = options.Multicore
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int, options.runs.split(','))
if options.PrepID:
schema['PrepID'] = options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1, ntask - 1):
schema['Task%d' % it]['KeepOutput'] = False
schema['TaskChain'] = ntask - 1
schema.pop('Task%d' % ntask)
if options.priority:
schema['RequestPriority'] = options.priority
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
schema = reqMgrClient.purgeClonedSchema(schema)
print "submitting"
if (options.to_stepchain
and (schema['RequestType'] == 'TaskChain')):
## transform the schema into StepChain schema
print "Transforming a TaskChain into a StepChain"
mcore = 0
mem = 0
schema['RequestType'] = 'StepChain'
schema['StepChain'] = schema.pop('TaskChain')
schema['SizePerEvent'] = 0
schema['TimePerEvent'] = 0
step = 1
s_n = {}
while True:
if 'Task%d' % step in schema:
sname = 'Step%d' % step
schema[sname] = schema.pop('Task%d' % step)
tmcore = schema[sname].pop('Multicore')
tmem = schema[sname].pop('Memory')
if mcore and tmcore != mcore:
wfi.sendLog(
'rejector',
'the conversion to stepchain encoutered different value of Multicore %d != %d'
% (tmcore, mcore))
sendLog(
'rejector',
'the conversion of %s to stepchain encoutered different value of Multicore %d != %d'
% (wfo.name, tmcore, mcore),
level='critical')
mcore = max(mcore, tmcore)
mem = max(mem, tmem)
schema[sname]['StepName'] = schema[sname].pop(
'TaskName')
s_n[schema[sname]['StepName']] = sname
if 'InputTask' in schema[sname]:
schema[sname]['InputStep'] = schema[sname].pop(
'InputTask')
eff = 1.
up_s = sname
while True:
## climb up a step. supposedely already all converted
up_s = s_n.get(
schema[up_s].get('InputStep', None), None)
if up_s:
## multiply with the efficiency
eff *= schema[up_s].get(
'FilterEfficiency', 1.)
else:
## or stop there
break
if not 'KeepOutput' in schema[sname]:
## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
schema[sname]['KeepOutput'] = False
schema['TimePerEvent'] += eff * schema[sname].pop(
'TimePerEvent')
schema['SizePerEvent'] += eff * schema[sname].pop(
'SizePerEvent')
step += 1
else:
break
schema['Multicore'] = mcore
schema['Memory'] = mem
print json.dumps(schema, indent=2)
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
msg = "Error in cloning {}".format(wfo.name)
print(msg)
wfi.sendLog('rejector', msg)
# Get the error message
time.sleep(5)
data = reqMgrClient.requestManagerPost(
url, "/reqmgr2/data/request", schema)
wfi.sendLog('rejector', data)
print json.dumps(schema, indent=2)
return
print newWorkflow
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfi.sendLog(
'rejector', 'Cloned into %s by unified operator %s' %
(newWorkflow, comment))
#wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False)
else:
wfo.status = 'trouble' if options.set_trouble else 'forget'
wfi.notifyRequestor('Rejected by unified operator %s' %
(comment),
do_batch=False)
session.commit()
else:
msg = "Error in rejecting {}: {}".format(wfo.name, results)
print(msg)
wfi.sendLog('rejector', msg)
def singleRecovery(url, task, initial, actions, do=False):
payload = {
"Requestor": os.getenv('USER'),
"Group": 'DATAOPS',
"RequestType": "Resubmission",
"ACDCServer": initial['ConfigCacheUrl'],
"ACDCDatabase": "acdcserver",
"OriginalRequestName": initial['RequestName'],
"OpenRunningTimeout": 0
}
copy_over = [
'PrepID', 'Campaign', 'RequestPriority', 'TimePerEvent',
'SizePerEvent', 'Group', 'Memory', 'RequestString', 'CMSSWVersion'
]
for c in copy_over:
if c in initial:
payload[c] = copy.deepcopy(initial[c])
else:
print c, "not in the initial payload"
#a massage ? boost the recovery over the initial wf
payload['RequestPriority'] *= 2
payload['RequestPriority'] = min(500000, payload['RequestPriority'])
if actions:
for action in actions:
#if action.startswith('split'):
# factor = int(action.split('-')[-1]) if '-' in action else 2
# print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
# ## mention it's taking 2 times longer to have a 2 times finer splitting
# payload['TimePerEvent'] = factor*payload['TimePerEvent']
if action.startswith('mem'):
arg = action.split('-', 1)[-1]
increase = set_to = None
tasks, set_to = arg.split(':') if ':' in arg else (None, arg)
tasks = tasks.split(',') if tasks else []
if set_to.startswith('+'):
increase = int(set_to[1:])
else:
set_to = int(set_to)
## increase the memory requirement by 1G
if 'TaskChain' in initial:
mem_dict = {}
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in initial:
tname = payload.setdefault(t,
initial[t])['TaskName']
mem = mem_dict.setdefault(tname,
payload[t]['Memory'])
if tasks and not tname in tasks:
print tname, "not concerned"
continue
if set_to:
mem_dict[tname] = set_to
else:
mem_dict[tname] += increase
else:
break
payload['Memory'] = mem_dict
else:
payload['Memory'] = set_to
#increase = int(action.split('-')[-1]) if '-' in action else 1000
## increase the memory requirement by 1G
#payload['Memory'] += increase
if action.startswith('split') and (
initial['RequestType'] in ['MonteCarlo'] or
(initial['RequestType'] in ['TaskChain']
and not 'InputDataset' in initial['Task1'])):
print "I should not be doing splitting for this type of request", initial[
'RequestName']
return None
if action.startswith('core'):
arg = action.split('-', 1)[-1]
tasks, set_to = arg.split(':') if ':' in arg else (None, arg)
tasks = tasks.split(',') if tasks else []
set_to = int(set_to)
if 'TaskChain' in initial:
core_dict = {}
mem_dict = payload['Memory'] if type(
payload['Memory']) == dict else {}
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in initial:
tname = payload.setdefault(t,
initial[t])['TaskName']
mcore = core_dict.setdefault(
tname, payload[t]['Multicore'])
mem = mem_dict.setdefault(tname,
payload[t]['Memory'])
if tasks and not tname in tasks:
print tname, "not concerned"
continue
factor = (set_to / float(mcore))
fraction_constant = 0.4
mem_per_core_c = int(
(1 - fraction_constant) * mem / float(mcore))
##scale the memory
mem_dict[tname] += (set_to -
mcore) * mem_per_core_c
## scale time/event
time_dict[
tname] = payload[t]['TimePerEvent'] / factor
## set the number of cores
core_dict[tname] = set_to
else:
break
payload['Multicore'] = core_dict
##payload['TimePerEvent'] = time_dict ## cannot be used yet
else:
payload['Multicore'] = increase
acdc_round = 0
initial_string = payload['RequestString']
if initial_string.startswith('ACDC'):
if initial_string[4].isdigit():
acdc_round = int(initial_string[4])
acdc_round += 1
#print acdc_round
#print "This is not allowed yet"
#return None
initial_string = initial_string.replace('ACDC_', '').replace(
'ACDC%d_' % (acdc_round - 1), '')
payload['RequestString'] = 'ACDC%d_%s' % (acdc_round, initial_string)
payload['InitialTaskPath'] = task
if not do:
print json.dumps(payload, indent=2)
return None
print "ACDC payload"
print json.dumps(payload, indent=2)
print actions
## submit
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error in making ACDC for", initial["RequestName"]
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error twice in making ACDC for", initial["RequestName"]
return None
## perform modifications
if actions:
for action in actions:
if action.startswith('split'):
factor = int(action.split('-')[-1]) if '-' in action else 2
acdcInfo = workflowInfo(url, acdc)
splittings = acdcInfo.getSplittings()
for split in splittings:
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split:
print "Changing %s (%d) by a factor %d" % (
act, split[act], factor),
split[act] /= factor
print "to", split[act]
break
split['requestName'] = acdc
print "changing the splitting of", acdc
print json.dumps(split, indent=2)
print reqMgrClient.setWorkflowSplitting(url, acdc, split)
data = reqMgrClient.setWorkflowApproved(url, acdc)
print data
return acdc
def singleClone(url, wfname, actions, comment, do=False):
wfi = workflowInfo(url, wfname)
payload = wfi.getSchema()
initial = wfi.request
payload['Requestor'] = os.getenv('USER')
payload['Group'] = 'DATAOPS'
payload['OriginalRequestName'] = initial['RequestName']
payload['RequestPriority'] = initial['RequestPriority']
if 'ProcessingVersion' in initial:
payload['ProcessingVersion'] = int(initial['ProcessingVersion']) +1
else:
payload['ProcessingVersion'] = 2
## drop parameters on the way to reqmgr2
paramBlacklist = ['BlockCloseMaxEvents', 'BlockCloseMaxFiles', 'BlockCloseMaxSize', 'BlockCloseMaxWaitTime',
'CouchWorkloadDBName', 'CustodialGroup', 'CustodialSubType', 'Dashboard',
'GracePeriod', 'HardTimeout', 'InitialPriority', 'inputMode', 'MaxMergeEvents', 'MaxMergeSize',
'MaxRSS', 'MaxVSize', 'MinMergeSize', 'NonCustodialGroup', 'NonCustodialSubType',
'OutputDatasets', 'ReqMgr2Only', 'RequestDate' 'RequestorDN', 'RequestName', 'RequestStatus',
'RequestTransition', 'RequestWorkflow', 'SiteWhitelist', 'SoftTimeout', 'SoftwareVersions',
'SubscriptionPriority', 'Team', 'timeStamp', 'TrustSitelists', 'TrustPUSitelists',
'TotalEstimatedJobs', 'TotalInputEvents', 'TotalInputLumis', 'TotalInputFiles','checkbox',
'DN', 'AutoApproveSubscriptionSites', 'NonCustodialSites', 'CustodialSites', 'OriginalRequestName', 'Teams', 'OutputModulesLFNBases',
'SiteBlacklist', 'AllowOpportunistic', '_id']
for p in paramBlacklist:
if p in payload:
payload.pop( p )
pass
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
if 'TaskChain' in payload:
print "Setting memory for clone of task chain"
it=1
while True:
t = 'Task%d'%it
it+=1
if t in payload:
payload[t]['Memory'] = actions[action]
print "Memory set for Task%d"%it
else:
break
else:
print "Setting memory for non-taskchain workflow"
payload['Memory'] = actions[action]
print "Memory set to " + actions[action]
#This line is doesn't work for some reason
# wfi.sendLog('actor','Memory of clone set to %d'%actions[action])
print "Clone payload"
# print json.dumps( payload , indent=2)
print actions
#Create clone
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error in making clone for",initial["RequestName"]
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error twice in making clone for",initial["RequestName"]
sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"],level='critical')
wfi.sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"])
return None
if actions:
for action in actions:
if action.startswith('split'):
cloneinfo = workflowInfo(url, clone)
splittings = cloneinfo.getSplittings()
if actions[action] != 'Same' and actions[action] != 'max' and actions[action] != '':
factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split:
wfi.sendLog('actor','Changing %s (%d) by a factor %d'%( act, split[act], factor))
print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
split[act] /= factor
print "to",split[act]
break
split['requestName'] = clone
print "changing the splitting of",clone
print json.dumps( split, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, clone, split )
elif 'max' in actions[action]:
for split in splittings:
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split:
wfi.sendLog('actor','Max splitting set for %s (%d'%( act, split[act]))
print "Changing %s (%d) "%( act, split[act]),
split[act] = 1
print "to max splitting ",split[act]
break
split['requestName'] = clone
print "changing the splitting of",clone
print json.dumps( split, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, clone, split )
#Approve
data = reqMgrClient.setWorkflowApproved(url, clone)
wfi.sendLog('actor','Cloned into %s'%clone)
# wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
# wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)
print data
return clone
def singleRecovery(url, task , initial, actions, do=False):
payload = {
"Requestor" : os.getenv('USER'),
"Group" : 'DATAOPS',
"RequestType" : "Resubmission",
"ACDCServer" : "https://cmsweb.cern.ch/couchdb",
"ACDCDatabase" : "acdcserver",
"OriginalRequestName" : initial['RequestName']
}
copy_over = ['PrepID','RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ]
for c in copy_over:
payload[c] = copy.deepcopy(initial[c])
if actions:
for action in actions:
#if action.startswith('split'):
# factor = int(action.split('-')[-1]) if '-' in action else 2
# print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
# ## mention it's taking 2 times longer to have a 2 times finer splitting
# payload['TimePerEvent'] = factor*payload['TimePerEvent']
if action.startswith('mem'):
increase = int(action.split('-')[-1]) if '-' in action else 1000
## increase the memory requirement by 1G
payload['Memory'] += increase
if payload['RequestString'].startswith('ACDC'):
print "This is not allowed yet"
return None
payload['RequestString'] = 'ACDC_'+payload['RequestString']
payload['InitialTaskPath'] = task
if not do:
print json.dumps( payload, indent=2)
return None
## submit
response = reqMgrClient.submitWorkflow(url, payload)
m = re.search("details\/(.*)\'",response)
if not m:
print "Error in making ACDC for",initial["RequestName"]
print response
response = reqMgrClient.submitWorkflow(url, payload)
m = re.search("details\/(.*)\'",response)
if not m:
print "Error twice in making ACDC for",initial["RequestName"]
print response
return None
acdc = m.group(1)
## perform modifications
if actions:
for action in actions:
if action.startswith('split'):
factor = int(action.split('-')[-1]) if '-' in action else 2
acdcInfo = workflowInfo(url, acdc)
splittings = acdcInfo.getSplittings()
for split in splittings:
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split:
print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
split[act] /= factor
print "to",split[act]
break
split['requestName'] = acdc
print "changing the splitting of",acdc
print json.dumps( split, indent=2 )
print reqMgrClient.setWorkflowSplitting(url, split )
data = reqMgrClient.setWorkflowApproved(url, acdc)
print data
return acdc
def rejector(url, specific, options=None):
up = componentInfo()
if specific and specific.startswith('/'):
return
if options.filelist:
wfs = []
for line in filter(None, open(options.filelist).read().split('\n')):
print line
wfs.extend( session.query(Workflow).filter(Workflow.name.contains(line)).all())
elif specific:
wfs = session.query(Workflow).filter(Workflow.name.contains(specific)).all()
else:
return
print len(wfs),"to reject"
if len(wfs)>1:
print "\n".join( [wfo.name for wfo in wfs] )
answer = raw_input('Reject these')
if not answer.lower() in ['y','yes']:
return
for wfo in wfs:
#wfo = session.query(Workflow).filter(Workflow.name == specific).first()
if not wfo:
print "cannot reject",spec
return
results=[]
wfi = workflowInfo(url, wfo.name)
datasets = set(wfi.request['OutputDatasets'])
reqMgrClient.invalidateWorkflow(url, wfo.name, current_status=wfi.request['RequestStatus'])
wfi.sendLog('rejector','invalidating the workflow by unified operator')
## need to find the whole familly and reject the whole gang
familly = getWorkflowById( url, wfi.request['PrepID'] , details=True)
for fwl in familly:
if fwl['RequestDate'] < wfi.request['RequestDate']: continue
if fwl['RequestType']!='Resubmission': continue
if 'OriginalRequestName' in fwl and fwl['OriginalRequestName'] != wfi.request['RequestName']: continue
print "rejecting",fwl['RequestName']
reqMgrClient.invalidateWorkflow(url, fwl['RequestName'], current_status=fwl['RequestStatus'])
datasets.update( fwl['OutputDatasets'] )
for dataset in datasets:
if options.keep:
print "keeping",dataset,"in its current status"
else:
results.append( setDatasetStatus(dataset, 'INVALID') )
#if wfi.request['RequestStatus'] in ['rejected','rejected-archived','aborted','aborted-archived']:
# print 'already',wfi.request['RequestStatus']
# if not options.clone:
# wfo.status = 'forget'
# session.commit()
# continue
if all(map(lambda result : result in ['None',None,True],results)):
wfo.status = 'forget'
session.commit()
print wfo.name,"and",datasets,"are rejected"
if options and options.clone:
schema = wfi.getSchema()
schema['Requestor'] = os.getenv('USER')
schema['Group'] = 'DATAOPS'
schema['OriginalRequestName'] = wfo.name
if 'ProcessingVersion' in schema:
schema['ProcessingVersion'] = int(schema['ProcessingVersion'])+1 ## dubious str->int conversion
else:
schema['ProcessingVersion']=2
for k in schema.keys():
if k.startswith('Team'):
schema.pop(k)
if k.startswith('checkbox'):
schema.pop(k)
## a few tampering of the original request
if options.Memory:
schema['Memory'] = options.Memory
if options.deterministic:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['DeterministicPileup'] = True
if options.EventsPerJob:
if schema['RequestType'] == 'TaskChain':
schema['Task1']['EventsPerJob'] = options.EventsPerJob
else:
schema['EventsPerJob'] = options.EventsPerJob
if options.EventAwareLumiBased:
schema['SplittingAlgo'] = 'EventAwareLumiBased'
if options.TimePerEvent:
schema['TimePerEvent'] = options.TimePerEvent
if options.ProcessingString:
schema['ProcessingString'] = options.ProcessingString
if options.AcquisitionEra:
schema['AcquisitionEra'] = options.AcquisitionEra
if options.runs:
schema['RunWhitelist'] = map(int,options.runs.split(','))
if options.PrepID:
schema['PrepID'] =options.PrepID
if schema['RequestType'] == 'TaskChain' and options.no_output:
ntask = schema['TaskChain']
for it in range(1,ntask-1):
schema['Task%d'%it]['KeepOutput'] = False
schema['TaskChain'] = ntask-1
schema.pop('Task%d'%ntask)
## update to the current priority
schema['RequestPriority'] = wfi.request['RequestPriority']
## drop shit on the way to reqmgr2
for p in ['RequestStatus',
'RequestTransition',
'RequestorDN',
'RequestWorkflow',
'OutputDatasets',
'ReqMgr2Only',
#'Group',
'RequestDate',
#'ConfigCacheUrl',
'RequestName',
'timeStamp',
'SoftwareVersions',
'CouchURL'
]:
if p in schema:
schema.pop( p )
#pass
print "submitting"
print json.dumps( schema, indent=2 )
newWorkflow = reqMgrClient.submitWorkflow(url, schema)
if not newWorkflow:
print "error in cloning",wfo.name
print json.dumps( schema, indent=2 )
return
print newWorkflow
#m = re.search("details\/(.*)\'",response)
#if not m:
# print "error in cloning",wfo.name
# print response
# print json.dumps( schema, indent=2 )
# return
#newWorkflow = m.group(1)
data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
print data
wfo.status = 'trouble'
session.commit()
wfi.sendLog('rejector','Cloned into %s by unified operator'%( newWorkflow ))
else:
print "error in rejecting",wfo.name,results
def singleClone(url, wfname, actions, comment, do=False):
wfi = workflowInfo(url, wfname)
payload = wfi.getSchema()
initial = wfi.request
payload['Requestor'] = os.getenv('USER')
payload['Group'] = 'DATAOPS'
payload['OriginalRequestName'] = initial['RequestName']
payload['RequestPriority'] = initial['RequestPriority']
if 'ProcessingVersion' in initial:
payload['ProcessingVersion'] = int(initial['ProcessingVersion']) + 1
else:
payload['ProcessingVersion'] = 2
payload = reqMgrClient.purgeClonedSchema(payload)
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[
action] != 'Same':
if 'TaskChain' in payload:
print "Setting memory for clone of task chain"
mem_dict = {}
it = 1
while True:
t = 'Task%d' % it
it += 1
if t in payload:
tname = payload[t]['TaskName']
mem_dict[tname] = int(actions[action])
print "Memory set for Task%d" % it
else:
break
payload['Memory'] = mem_dict
else:
print "Setting memory for non-taskchain workflow"
payload['Memory'] = int(actions[action])
print "Memory set to " + actions[action]
print "Clone payload"
# print json.dumps( payload , indent=2)
print actions
#Create clone
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error in making clone for", initial["RequestName"]
clone = reqMgrClient.submitWorkflow(url, payload)
if not clone:
print "Error twice in making clone for", initial["RequestName"]
sendLog('actor',
'Failed to make a clone twice for %s!' %
initial["RequestName"],
level='critical')
wfi.sendLog(
'actor', 'Failed to make a clone twice for %s!' %
initial["RequestName"])
return None
if actions:
for action in actions:
if action.startswith('split'):
cloneinfo = workflowInfo(url, clone)
splittings = cloneinfo.getSplittingsNew(strip=True)
if actions[action] != 'Same' and actions[
action] != 'max' and actions[action] != '':
factor = int(
actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
split_par = split['splitParams']
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split_par:
wfi.sendLog(
'actor',
'Changing %s (%d) by a factor %d' %
(act, split_par[act], factor))
split_par[act] /= factor
print "to", split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
elif 'max' in actions[action]:
for split in splittings:
split_par = split['splitParams']
for act in [
'avg_events_per_job', 'events_per_job',
'lumis_per_job'
]:
if act in split_par:
wfi.sendLog(
'actor', 'Max splitting set for %s (%d' %
(act, split_par[act]))
print "Changing %s (%d) " % (act,
split_par[act]),
split_par[act] = 1
print "to max splitting ", split_par[act]
break
#split['requestName'] = clone
#print "changing the splitting of",clone
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, clone, split )
print "changing the splitting of", clone
print json.dumps(splittings, indent=2)
print reqMgrClient.setWorkflowSplitting(url, clone, splittings)
#Approve
data = reqMgrClient.setWorkflowApproved(url, clone)
#wfi.sendLog('actor','Cloned into %s'%clone)
# wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
# wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)
print data
return clone
def singleRecovery(url, task, initial, actions, do=False):
print "Inside single recovery!"
payload = {
"Requestor" : os.getenv('USER'),
"Group" : 'DATAOPS',
"RequestType" : "Resubmission",
"ACDCServer" : initial['ConfigCacheUrl'],
"ACDCDatabase" : "acdcserver",
"OriginalRequestName" : initial['RequestName'],
"OpenRunningTimeout" : 0
}
copy_over = ['PrepID','Campaign','RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ,'CMSSWVersion']
for c in copy_over:
if c in initial:
payload[c] = copy.deepcopy(initial[c])
else:
print c,"not in the initial payload"
#a massage ? boost the recovery over the initial wf
# payload['RequestPriority'] *= 10
#Max priority is 1M
payload['RequestPriority'] = min(500000, payload['RequestPriority']*2 ) ## never above 500k
#change parameters based on actions here
if actions:
for action in actions:
if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
#if multicore parameter is also used, need to scale memory by the new number of cores
if 'multicore' in actions and actions['multicore'] != "":
continue
## Taskchains needs to be treated special to set the memory to all tasks
set_to = int(actions[action])
if 'TaskChain' in initial:
mem_dict = {}
it = 1
while True:
t = 'Task%d'%it
it += 1
if t in initial:
tname = payload.setdefault(t, initial[t])['TaskName']
mem = mem_dict.setdefault( tname, payload[t]['Memory'])
mem_dict[tname] = set_to
else:
break
payload['Memory'] = mem_dict
print "Memory set to: ",json.dumps( mem_dict, indent=2)
else:
payload['Memory'] = set_to
print "Memory set to: ", set_to
if action.startswith('multicore') and actions[action] != "":
set_to = int(actions[action] )
## Taskchains needs to be treated special to set the multicore and memory values to all tasks
if 'TaskChain' in initial:
mem_dict = payload['Memory'] if type(payload['Memory'])==dict else {}
core_dict = {}
it = 1
while True:
t = 'Task%d'%it
it += 1
if t in initial:
tname = payload.setdefault(t, initial[t])['TaskName']
mem = mem_dict.setdefault( tname, payload[t]['Memory'])
#Need to scale the memory by the new number of cores
initial_cores = payload[t].setdefault('Multicore', 1)
if 'memory' in actions and actions['memory'] != "" and actions['memory'] != 'Same':
mem = actions['memory']
fraction_constant = 0.4
mem_per_core_c = int (( 1 - fraction_constant) * mem / float(initial_cores) )
mem_dict[tname] = int ( mem + (set_to - initial_cores)*mem_per_core_c )
core_dict[tname] = set_to
print "For ", t
print "Multicore set to ", set_to
print "Memory set to ", mem_dict[tname]
else:
break
payload['Memory'] = mem_dict
payload['Multicore'] = core_dict
else:
#Need to scale the memory by the new number of cores
initial_cores = initial.setdefault('Multicore', 1)
mem = payload['Memory']
if 'memory' in actions and actions['memory'] != "" and actions['memory'] != 'Same' :
mem = actions['memory']
fraction_constant = 0.4
mem_per_core_c = int (( 1 - fraction_constant) * mem / float(initial_cores) )
payload['Multicore'] = set_to
payload['Memory'] = int ( mem + (set_to - initial_cores)*mem_per_core_c )
print "Multicore set to ", set_to
print "Memory set to ", payload['Memory']
if action.startswith('split'):
split_alert = (initial['RequestType'] in ['MonteCarlo'] )
for key in initial:
if key == 'SplittingAlgo' and (initial[key] in ['EventBased']):
split_alert = True
elif key.startswith('Task') and key != 'TaskChain':
for key2 in initial[key]:
if key2 == 'TaskName':
this_taskname = initial[key][key2]
recover_task = task.split('/')[-1]
print "For recovery of task",recover_task
print "Looking at task",this_taskname
if (recover_task == this_taskname) and (initial[key]['SplittingAlgo'] in ['EventBased']):
## the task to be recovered is actually of the wrong type to allow change of splitting
sendLog('actor','To recover on %s, changing the splitting on %s is not really allowed and this will be ignored instead of failing acdc.'%( task, initial[key]['SplittingAlgo']), level='critical')
## do not send an alert and stop the acdc
#split_alert = True
if split_alert:
sendLog('actor','Cannot change splitting for %s'%initial['RequestName'],level='critical')
print "I should not be doing splitting for this type of request",initial['RequestName']
return None
acdc_round = 0
initial_string = payload['RequestString']
if initial_string.startswith('ACDC'):
if initial_string[4].isdigit():
acdc_round = int(initial_string[4])
acdc_round += 1
initial_string = initial_string.replace('ACDC_','').replace('ACDC%d_'%(acdc_round-1),'')
payload['RequestString'] = 'ACDC%d_%s'%(acdc_round,initial_string)
payload['InitialTaskPath'] = task
if not do:
print json.dumps( payload, indent=2)
return None
print "ACDC payload"
# print json.dumps( payload , indent=2)
print actions
## submit here
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error in making ACDC for",initial["RequestName"]
acdc = reqMgrClient.submitWorkflow(url, payload)
if not acdc:
print "Error twice in making ACDC for",initial["RequestName"]
sendLog('actor','Failed twice in making ACDCs for %s!'%initial['RequestName'],level='critical')
return None
## change splitting if requested
if actions:
for action in actions:
if action.startswith('split'):
acdcInfo = workflowInfo(url, acdc)
splittings = acdcInfo.getSplittingsNew(strip=True)
if actions[action] != 'Same' and actions[action] != 'max':
factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
for split in splittings:
split_par = split['splitParams']
if split['splitAlgo'] in ['EventBased']:
sendLog('actor',"Changing the splitting on %s for %s is not permitted. Not changing."%(split['splitAlgo'],initial["RequestName"]), level='critical')
continue
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split_par:
print "Changing %s (%d) by a factor %d"%( act, split_par[act], factor),
split_par[act] /= factor
print "to",split_par[act]
break
#split['requestName'] = acdc
#print "changing the splitting of",acdc
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, acdc, split )
elif 'max' in actions[action]:
for split in splittings:
split_par = split['splitParams']
for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
if act in split_par:
print "Changing %s (%d) "%( act, split_par[act]),
split_par[act] = 1
print "to max splitting ",split_par[act]
break
#split['requestName'] = acdc
#print "changing the splitting of",acdc
#print json.dumps( split, indent=2 )
#print reqMgrClient.setWorkflowSplitting(url, acdc, split )
print "changing the splitting of",acdc
print json.dumps( splittings, indent=2 )
done = reqMgrClient.setWorkflowSplitting(url, acdc, splittings )
## check on done == True
data = reqMgrClient.setWorkflowApproved(url, acdc)
print data
return acdc