Beispiel #1
0
def cloneWorkflow(workflow, user, group, verbose=True, backfill=False, testbed=False, memory=None, bwl=None):
    """
    clones a workflow
    """
    # Get info about the workflow to be cloned
    helper = reqMgrClient.retrieveSchema(workflow, reqmgrCouchURL)
    # Adapt schema and add original request to it
    try:
        cache = reqMgrClient.getWorkloadCache(url, workflow)
    except:
        cache = None

    schema = modifySchema(helper, workflow, user, group, cache, None, None, backfill, memory)

    schema['OriginalRequestName'] = workflow
    if verbose:
        pprint(schema)
    
    if bwl:
        if 'Task1' in schema:
            schema['Task1']['BlockWhitelist'] = bwl.split(',')
        else:
            schema['BlockWhitelist'] = bwl.split(',')
    print 'Submitting workflow'
    # Submit cloned workflow to ReqMgr
    if testbed:
        newWorkflow = reqMgrClient.submitWorkflow(url_tb, schema)
    else:
        newWorkflow = reqMgrClient.submitWorkflow(url, schema)
    if verbose:
        print "RESPONSE", newWorkflow

    # find the workflow name in response
    if newWorkflow:
        print 'Cloned workflow: ' + newWorkflow
        if verbose:
            print newWorkflow
            print 'Approving request response:'
        # TODO only for debug
        #response = reqMgrClient.setWorkflowSplitting(url, schema)
        # print "RESPONSE", response
        #schema['requestName'] = requestName
        #schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
        #schema['splittingAlgo'] = splittingAlgo

        # Move the request to Assignment-approved
        if testbed:
            data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
        else:
            data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        if verbose:
            print data
        # return the name of new workflow
        return newWorkflow
    else:
        if verbose:
            print newWorkflow
        else:
            print "Couldn't clone the workflow."
        return None
Beispiel #2
0
def cloneWorkflow(workflow, user, group, verbose=True, backfill=False, testbed=False, memory=None, bwl=None):
    """
    clones a workflow
    """
    # Get info about the workflow to be cloned
    helper = reqMgrClient.retrieveSchema(workflow, reqmgrCouchURL)
    # Adapt schema and add original request to it
    try:
        cache = reqMgrClient.getWorkloadCache(url, workflow)
    except:
        cache = None

    schema = modifySchema(helper, workflow, user, group, cache, None, None, backfill, memory)

    schema['OriginalRequestName'] = workflow
    if verbose:
        pprint(schema)
    
    if bwl:
        if 'Task1' in schema:
            schema['Task1']['BlockWhitelist'] = bwl.split(',')
        else:
            schema['BlockWhitelist'] = bwl.split(',')
    print 'Submitting workflow'
    # Submit cloned workflow to ReqMgr
    if testbed:
        newWorkflow = reqMgrClient.submitWorkflow(url_tb, schema, reqmgr2 = True)
    else:
        newWorkflow = reqMgrClient.submitWorkflow(url, schema, reqmgr2 = True)
    if verbose:
        print "RESPONSE", newWorkflow

    # find the workflow name in response
    if newWorkflow:
        print 'Cloned workflow: ' + newWorkflow
        if verbose:
            print newWorkflow
            print 'Approving request response:'
        # TODO only for debug
        #response = reqMgrClient.setWorkflowSplitting(url, schema)
        # print "RESPONSE", response
        #schema['requestName'] = requestName
        #schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
        #schema['splittingAlgo'] = splittingAlgo

        # Move the request to Assignment-approved
        if testbed:
            data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
        else:
            data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        if verbose:
            print data
        # return the name of new workflow
        return newWorkflow
    else:
        if verbose:
            print newWorkflow
        else:
            print "Couldn't clone the workflow."
        return None
Beispiel #3
0
def cloneWorkflow(workflow, user, group, verbose=True, backfill=False, testbed=False, memory=None, timeperevent=None, bwl=None):
    """
    clones a workflow
    """
    # Adapt schema and add original request to it
    cache = reqMgrClient.getWorkflowInfo(url, workflow)

    schema = modifySchema(cache, workflow, user, group, None, None, backfill, memory, timeperevent)

    if verbose:
        pprint(schema)
    
    if bwl:
        if 'Task1' in schema:
            schema['Task1']['BlockWhitelist'] = bwl.split(',')
        else:
            schema['BlockWhitelist'] = bwl.split(',')
    ## only once
    ####schema['CMSSWVersion'] = 'CMSSW_8_0_16'

    print 'Submitting workflow'
    # Submit cloned workflow to ReqMgr
    if testbed:
        newWorkflow = reqMgrClient.submitWorkflow(url_tb, schema)
    else:
        newWorkflow = reqMgrClient.submitWorkflow(url, schema)
    if verbose:
        print "RESPONSE", newWorkflow

    # find the workflow name in response
    if newWorkflow:
        print 'Cloned workflow: ' + newWorkflow
        if verbose:
            print newWorkflow
            print 'Approving request response:'
        # TODO only for debug
        #response = reqMgrClient.setWorkflowSplitting(url, schema)
        # print "RESPONSE", response
        #schema['requestName'] = requestName
        #schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
        #schema['splittingAlgo'] = splittingAlgo

        # Move the request to Assignment-approved
        if testbed:
            data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
        else:
            data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        if verbose:
            print data
        # return the name of new workflow
        return newWorkflow
    else:
        if verbose:
            print newWorkflow
        else:
            print "Couldn't clone the workflow."
        return None
def cloneWorkflow(workflow, user, group, verbose=False, backfill=False, testbed=False):
    """
    clones a workflow
    """
    # Get info about the workflow to be cloned
    helper = reqMgrClient.retrieveSchema(workflow, reqmgrCouchURL)
    # Adapt schema and add original request to it
    try:
        cache = reqMgrClient.getWorkloadCache(url, workflow)
    except:
        cache = None
        
    schema = modifySchema(helper, user, group, cache, backfill)
    schema['OriginalRequestName'] = workflow
    if verbose:
        pprint(schema)
    print 'Submitting workflow'
    # Sumbit cloned workflow to ReqMgr
    if testbed:
        response = reqMgrClient.submitWorkflow(url_tb, schema)
    else:
        response = reqMgrClient.submitWorkflow(url, schema)
    if verbose:
        print "RESPONSE", response

    # find the workflow name in response
    m = re.search("details\/(.*)\'", response)
    if m:
        newWorkflow = m.group(1)
        print 'Cloned workflow: ' + newWorkflow
        if verbose:
            print response
            print 'Approving request response:'
        # TODO only for debug
        #response = reqMgrClient.setWorkflowSplitting(url, schema)
        # print "RESPONSE", response
        #schema['requestName'] = requestName
        #schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
        #schema['splittingAlgo'] = splittingAlgo

        # Move the request to Assignment-approved
        if testbed:
            data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
        else:
            data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        if verbose:
            print data
        # return the name of new workflow
        return newWorkflow
    else:
        if verbose:
            print response
        else:
            print "Couldn't clone the workflow."
        return None
Beispiel #5
0
def cloneWorkflow(workflow, user, group, verbose=False):
    """
    clones a workflow
    """
    # Get info about the workflow to be cloned
    helper = retrieveSchema(workflow)
    schema = modifySchema(helper, user, group)

    print 'Submitting workflow'
    # Sumbit cloned workflow to ReqMgr
    response = reqMgrClient.submitWorkflow(url,schema)
    #find the workflow name in response
    m = re.search("details\/(.*)\'",response)
    if m:
        newWorkflow = m.group(1)
        if verbose:
            print 'Cloned workflow: '+newWorkflow
            print response    

            print 'Approve request response:'
        # Move the request to Assignment-approved
        data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        if verbose:
            print data
        #return the name of new workflow
        return newWorkflow
    else:
        if verbose:
            print response
        return None
Beispiel #6
0
def rejector(url, specific, options=None):

    if specific.startswith('/'):
        pass
    else:
        wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject", spec
            return
        results = []
        wfi = workflowInfo(url, wfo.name)
        if wfi.request['RequestStatus'] in [
                'assignment-approved', 'new', 'completed'
        ]:
            #results.append( reqMgrClient.rejectWorkflow(url, wfo.name))
            reqMgrClient.rejectWorkflow(url, wfo.name)
        else:
            #results.append( reqMgrClient.abortWorkflow(url, wfo.name))
            reqMgrClient.abortWorkflow(url, wfo.name)

        datasets = wfi.request['OutputDatasets']
        for dataset in datasets:
            if options.keep:
                print "keeping", dataset, "in its current status"
            else:
                results.append(
                    setDatasetStatusDBS3.setStatusDBS3(
                        'https://cmsweb.cern.ch/dbs/prod/global/DBSWriter',
                        dataset, 'INVALID', None))

        if all(map(lambda result: result in ['None', None], results)):
            wfo.status = 'forget'
            session.commit()
            print wfo.name, "and", datasets, "are rejected"
            if options and options.clone:
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion'] += 1
                else:
                    schema['ProcessingVersion'] = 2
                ##schema.pop('RequestDate') ## ok then, let's not reset the time stamp
                if options.Memory:
                    schema['Memory'] = options.Memory
                response = reqMgrClient.submitWorkflow(url, schema)
                m = re.search("details\/(.*)\'", response)
                if not m:
                    print "error in cloning", wfo.name
                    print response
                    return
                newWorkflow = m.group(1)
                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfo.status = 'trouble'
                session.commit()
        else:
            print "error in rejecting", wfo.name, results
def rejector(url, specific, options=None):

    up = componentInfo()

    if specific.startswith('/'):
        pass
    else:
        wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject",spec
            return
        results=[]
        wfi = workflowInfo(url, wfo.name)
        reqMgrClient.invalidateWorkflow(url, wfo.name, current_status=wfi.request['RequestStatus'])
        #if wfi.request['RequestStatus'] in ['assignment-approved','new','completed']:
        #    #results.append( reqMgrClient.rejectWorkflow(url, wfo.name))
        #    reqMgrClient.rejectWorkflow(url, wfo.name)
        #else:
        #    #results.append( reqMgrClient.abortWorkflow(url, wfo.name))
        #    reqMgrClient.abortWorkflow(url, wfo.name)
        
        datasets = wfi.request['OutputDatasets']
        for dataset in datasets:
            if options.keep:
                print "keeping",dataset,"in its current status"
            else:
                results.append( setDatasetStatus(dataset, 'INVALID') )

        if all(map(lambda result : result in ['None',None,True],results)):
            wfo.status = 'forget'
            session.commit()
            print wfo.name,"and",datasets,"are rejected"
            if options and options.clone:
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                schema['OriginalRequestName'] = wfo.name
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion']+=1
                else:
                    schema['ProcessingVersion']=2
                ##schema.pop('RequestDate') ## ok then, let's not reset the time stamp
                if options.Memory:
                    schema['Memory'] = options.Memory
                response = reqMgrClient.submitWorkflow(url, schema)
                m = re.search("details\/(.*)\'",response)
                if not m:
                    print "error in cloning",wfo.name
                    print response
                    return 
                newWorkflow = m.group(1)
                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfo.status = 'trouble'
                session.commit()
        else:
            print "error in rejecting",wfo.name,results
Beispiel #8
0
def extendWorkflow(workflow,
                   user,
                   group,
                   verbose=False,
                   events=None,
                   firstlumi=None):

    if events is None:
        events = getMissingEvents(workflow)
    events = int(events)

    if firstlumi is None:
        #get the last lumi of the dataset
        dataset = reqMgrClient.outputdatasetsWorkflow(url, workflow).pop()

        lastLumi = dbs3Client.getMaxLumi(dataset)
        firstlumi = lastLumi
    firstlumi = int(firstlumi)

    # Get info about the workflow to be cloned
    cache = reqMgrClient.getWorkflowInfo(url, workflow)

    schema = modifySchema(cache, workflow, user, group, events, firstlumi,
                          None)
    if verbose:
        pprint(schema)
    print 'Submitting workflow'
    # Submit cloned workflow to ReqMgr
    response = reqMgrClient.submitWorkflow(url, schema)
    if verbose:
        print "RESPONSE", response

    #find the workflow name in response
    m = re.search("details\/(.*)\'", response)
    if m:
        newWorkflow = m.group(1)
        print 'Cloned workflow: ' + newWorkflow
        print 'Extended with', events, 'events'
        print response

        # Move the request to Assignment-approved
        print 'Approve request response:'
        data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        print data
    else:
        print response
def cloneWorkflow(workflow, user, group, verbose=False, backfill=False):
    """
    clones a workflow
    """
    # Get info about the workflow to be cloned
    helper = reqMgrClient.retrieveSchema(workflow)
    # get info from reqMgr
    schema = modifySchema(helper, user, group, backfill)
    if verbose:
        pprint(schema)
    print 'Submitting workflow'

    # Sumbit cloned workflow to ReqMgr
    response = reqMgrClient.submitWorkflow(url,schema)
    if verbose:
        print "RESPONSE", response
    
    #find the workflow name in response
    m = re.search("details\/(.*)\'",response)
    if m:
        newWorkflow = m.group(1)
        print 'Cloned workflow: '+newWorkflow
        if verbose:
            print response    
            print 'Approving request response:'
        #TODO only for debug
        #response = reqMgrClient.setWorkflowSplitting(url, schema)
        #print "RESPONSE", response
        #schema['requestName'] = requestName
        #schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
        #schema['splittingAlgo'] = splittingAlgo

        # Move the request to Assignment-approved
        data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        if verbose:
            print data
        #return the name of new workflow
        return newWorkflow
    else:
        if verbose:
            print response
        else:
            print "Couldn't clone the workflow."
        return None
Beispiel #10
0
def cloneWorkflow(workflow, user, group, verbose=False, backfill=False):
    """
    clones a workflow
    """
    # Get info about the workflow to be cloned
    helper = reqMgrClient.retrieveSchema(workflow, reqmgrCouchURL)
    # get info from reqMgr
    schema = modifySchema(helper, user, group, backfill)
    if verbose:
        pprint(schema)
    print 'Submitting workflow'

    # Sumbit cloned workflow to ReqMgr
    response = reqMgrClient.submitWorkflow(url, schema)
    if verbose:
        print "RESPONSE", response

    #find the workflow name in response
    m = re.search("details\/(.*)\'", response)
    if m:
        newWorkflow = m.group(1)
        print 'Cloned workflow: ' + newWorkflow
        if verbose:
            print response
            print 'Approving request response:'
        #TODO only for debug
        #response = reqMgrClient.setWorkflowSplitting(url, schema)
        #print "RESPONSE", response
        #schema['requestName'] = requestName
        #schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
        #schema['splittingAlgo'] = splittingAlgo

        # Move the request to Assignment-approved
        data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        if verbose:
            print data
        #return the name of new workflow
        return newWorkflow
    else:
        if verbose:
            print response
        else:
            print "Couldn't clone the workflow."
        return None
def extendWorkflow(workflow, user, group, verbose=False, events=None, firstlumi=None):
    
    if events is None:
        events = getMissingEvents(workflow)
    events = int(events)
    
    if firstlumi is None:
        #get the last lumi of the dataset
        dataset = reqMgrClient.outputdatasetsWorkflow(url, workflow).pop()
          
        lastLumi = dbs3Client.getMaxLumi(dataset)
        firstlumi = lastLumi
    firstlumi = int(firstlumi)
    
    # Get info about the workflow to be cloned
    helper = reqMgrClient.retrieveSchema(workflow)
    schema = modifySchema(helper, workflow, user, group, events, firstlumi)
    schema['OriginalRequestName'] = workflow
    if verbose:
        pprint(schema)
    print 'Submitting workflow'
    # Sumbit cloned workflow to ReqMgr
    response = reqMgrClient.submitWorkflow(url,schema)
    if verbose:
        print "RESPONSE", response
    
    #find the workflow name in response
    m = re.search("details\/(.*)\'",response)
    if m:
        newWorkflow = m.group(1)
        print 'Cloned workflow: '+newWorkflow
        print 'Extended with', events, 'events'
        print response
        
        # Move the request to Assignment-approved
        print 'Approve request response:'
        data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        print data
    else:
        print response
    pass
def main():
    # Check the arguements, get info from them
    if len(sys.argv) < 4:
        print "Usage:"
        print "  ./resubmit WORKFLOW_NAME USER GROUP [EVENTS]"
        sys.exit(0)
    workflow = sys.argv[1]
    user = sys.argv[2]
    group = sys.argv[3]
    if len(sys.argv) > 4:
        events = int(sys.argv[4])
    else:
        events = getMissingEvents(workflow)

    # Get info about the workflow to be cloned
    helper = reqMgrClient.retrieveSchema(workflow)
    schema = modifySchema(helper, workflow, user, group, events)

    print 'Submitting workflow'
    # Sumbit cloned workflow to ReqMgr
    response = reqMgrClient.submitWorkflow(url,schema)
    #find the workflow name in response
    m = re.search("details\/(.*)\'",response)
    if m:
        newWorkflow = m.group(1)
        print 'Cloned workflow: '+newWorkflow
        print 'Extended with', events, 'events'
        print response
        
        # Move the request to Assignment-approved
        print 'Approve request response:'
        data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        print data
    else:
        print response
    sys.exit(0)
Beispiel #13
0
def cloneWorkflow(workflow,
                  user,
                  group,
                  verbose=True,
                  backfill=False,
                  testbed=False,
                  memory=None,
                  timeperevent=None,
                  bwl=None):
    """
    clones a workflow
    """
    # Adapt schema and add original request to it
    cache = reqMgrClient.getWorkflowInfo(url, workflow)

    schema = modifySchema(cache, workflow, user, group, None, None, backfill,
                          memory, timeperevent)

    if verbose:
        pprint(schema)

    if bwl:
        if 'Task1' in schema:
            schema['Task1']['BlockWhitelist'] = bwl.split(',')
        else:
            schema['BlockWhitelist'] = bwl.split(',')
    ## only once
    ####schema['CMSSWVersion'] = 'CMSSW_8_0_16'

    print 'Submitting workflow'
    # Submit cloned workflow to ReqMgr
    if testbed:
        newWorkflow = reqMgrClient.submitWorkflow(url_tb, schema)
    else:
        newWorkflow = reqMgrClient.submitWorkflow(url, schema)
    if verbose:
        print "RESPONSE", newWorkflow

    # find the workflow name in response
    if newWorkflow:
        print 'Cloned workflow: ' + newWorkflow
        if verbose:
            print newWorkflow
            print 'Approving request response:'
        # TODO only for debug
        #response = reqMgrClient.setWorkflowSplitting(url, schema)
        # print "RESPONSE", response
        #schema['requestName'] = requestName
        #schema['splittingTask'] = '/%s/%s' % (requestName, taskName)
        #schema['splittingAlgo'] = splittingAlgo

        # Move the request to Assignment-approved
        if testbed:
            data = reqMgrClient.setWorkflowApproved(url_tb, newWorkflow)
        else:
            data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
        if verbose:
            print data
        # return the name of new workflow
        return newWorkflow
    else:
        if verbose:
            print newWorkflow
        else:
            print "Couldn't clone the workflow."
        return None
Beispiel #14
0
def rejector(url, specific, options=None):

    #use_mcm = True
    #up = componentInfo(mcm=use_mcm, soft=['mcm'])
    up = componentInfo()
    if not up.check(): return
    #use_mcm = up.status['mcm']
    #mcm = McMClient(dev=False) if use_mcm else None

    if specific and specific.startswith('/'):
        ## this is for a dataset
        print setDatasetStatus(specific, 'INVALID')
        return

    if options.filelist:
        wfs = []
        for line in filter(None, open(options.filelist).read().split('\n')):
            print line
            wfs.extend( session.query(Workflow).filter(Workflow.name.contains(line)).all())
    elif specific:
        wfs = session.query(Workflow).filter(Workflow.name.contains(specific)).all()
    else:
        wfs = session.query(Workflow).filter(Workflow.status == 'assistance-clone').all()
        #wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
        ## be careful then on clone case by case
        options.clone = True
        print "not supposed to function yet"
        return 

    print len(wfs),"to reject"

    if len(wfs)>1:
        print "\n".join( [wfo.name for wfo in wfs] )
        answer = raw_input('Reject these')
        if not answer.lower() in ['y','yes']:
            return
        
    for wfo in wfs:
        #wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject",spec
            return
        results=[]
        wfi = workflowInfo(url, wfo.name)

        datasets = set(wfi.request['OutputDatasets'])
        reqMgrClient.invalidateWorkflow(url, wfo.name, current_status=wfi.request['RequestStatus'])

        comment=""
        if options.comments: comment = ", reason: "+options.comments

        wfi.sendLog('rejector','invalidating the workflow by unified operator%s'%comment)
        ## need to find the whole familly and reject the whole gang
        familly = getWorkflowById( url, wfi.request['PrepID'] , details=True)
        for fwl in familly:
            if fwl['RequestDate'] < wfi.request['RequestDate']: continue
            if fwl['RequestType']!='Resubmission': continue
            ## does not work on second order acd
            if 'OriginalRequestName' in fwl and fwl['OriginalRequestName'] != wfi.request['RequestName']: continue
            print "rejecting",fwl['RequestName']
            reqMgrClient.invalidateWorkflow(url, fwl['RequestName'], current_status=fwl['RequestStatus'], cascade=False)
            datasets.update( fwl['OutputDatasets'] )

        for dataset in datasets:
            if options.keep:
                print "keeping",dataset,"in its current status"
            else:
                results.append( setDatasetStatus(dataset, 'INVALID') )
                pass


        if all(map(lambda result : result in ['None',None,True],results)):
            print wfo.name,"and",datasets,"are rejected"
            if options and options.clone:
                wfo.status = 'trouble'
                session.commit()                
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                schema['OriginalRequestName'] = wfo.name
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion'] = int(schema['ProcessingVersion'])+1 ## dubious str->int conversion
                else:
                    schema['ProcessingVersion']=2
                for k in schema.keys():
                    if k.startswith('Team'):
                        schema.pop(k)
                    if k.startswith('checkbox'):
                        schema.pop(k)

                ## a few tampering of the original request
                if options.Memory:
                    if schema['RequestType'] == 'TaskChain':
                        it=1
                        while True:
                            t = 'Task%d'%it
                            it+=1
                            if t in schema:
                                schema[t]['Memory'] = options.Memory
                            else:
                                break
                    else:
                        schema['Memory'] = options.Memory
                        
                if options.Multicore:
                    ## to do : set it properly in taslchains
                    schema['Multicore'] = options.Multicore

                if options.deterministic:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['DeterministicPileup']  = True
                if options.EventsPerJob:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['EventsPerJob'] = options.EventsPerJob
                    else:
                        schema['EventsPerJob'] = options.EventsPerJob
                if options.EventAwareLumiBased:
                    schema['SplittingAlgo'] = 'EventAwareLumiBased'
                if options.TimePerEvent:
                    schema['TimePerEvent'] = options.TimePerEvent

                if options.ProcessingString:
                    schema['ProcessingString'] = options.ProcessingString
                if options.AcquisitionEra:
                    schema['AcquisitionEra'] = options.AcquisitionEra
                if options.runs:
                    schema['RunWhitelist'] = map(int,options.runs.split(','))
                if options.PrepID:
                    schema['PrepID'] =options.PrepID

                if schema['RequestType'] == 'TaskChain' and options.no_output:
                    ntask = schema['TaskChain']
                    for it in range(1,ntask-1):
                        schema['Task%d'%it]['KeepOutput'] = False
                    schema['TaskChain'] = ntask-1
                    schema.pop('Task%d'%ntask)

                ## update to the current priority
                schema['RequestPriority'] = wfi.request['RequestPriority']

                ## drop shit on the way to reqmgr2
                paramBlacklist = ['BlockCloseMaxEvents', 'BlockCloseMaxFiles', 'BlockCloseMaxSize', 'BlockCloseMaxWaitTime',
                                  'CouchWorkloadDBName', 'CustodialGroup', 'CustodialSubType', 'Dashboard',
                                  'GracePeriod', 'HardTimeout', 'InitialPriority', 'inputMode', 'MaxMergeEvents', 'MaxMergeSize',
                                  'MaxRSS', 'MaxVSize', 'MinMergeSize', 'NonCustodialGroup', 'NonCustodialSubType',
                                  'OutputDatasets', 'ReqMgr2Only', 'RequestDate' 'RequestorDN', 'RequestName', 'RequestStatus',
                                  'RequestTransition', 'RequestWorkflow', 'SiteWhitelist', 'SoftTimeout', 'SoftwareVersions',
                                  'SubscriptionPriority', 'Team', 'timeStamp', 'TrustSitelists', 'TrustPUSitelists',
                                  'TotalEstimatedJobs', 'TotalInputEvents', 'TotalInputLumis', 'TotalInputFiles']
                for p in paramBlacklist:
                    if p in schema:
                        schema.pop( p )
                        #pass
                print "submitting"
                if (options.to_stepchain and (schema['RequestType']=='TaskChain')):
                    ## transform the schema into StepChain schema
                    print "Transforming a TaskChain into a StepChain"
                    schema['RequestType'] = 'StepChain'
                    schema['StepChain'] = schema.pop('TaskChain')
                    step=1
                    while True:
                        if 'Task%d'%step in schema:
                            schema['Step%d'%step] = schema.pop('Task%d'%step)
                            schema['Step%d'%step]['StepName'] = schema['Step%d'%step].pop('TaskName')
                            if 'InputTask' in schema['Step%d'%step]:
                                schema['Step%d'%step]['InputStep'] = schema['Step%d'%step].pop('InputTask')
                            if not 'KeepOutput' in schema['Step%d'%step]:
                                ## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
                                schema['Step%d'%step]['KeepOutput'] = False
                            step+=1
                        else:
                            break


                print json.dumps( schema, indent=2 )
                newWorkflow = reqMgrClient.submitWorkflow(url, schema)
                if not newWorkflow:
                    print "error in cloning",wfo.name
                    print json.dumps( schema, indent=2 )
                    return 
                print newWorkflow

                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfi.sendLog('rejector','Cloned into %s by unified operator %s'%( newWorkflow, comment ))
                wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False)
            else:
                wfo.status = 'forget'
                wfi.notifyRequestor('Rejected by unified operator %s'%( comment ),do_batch=False)
                session.commit()

        else:
            print "error in rejecting",wfo.name,results
Beispiel #15
0
def spawn_harvesting(url, wfi , in_full):
    SI = global_SI()

    all_OK = {}
    requests = []
    outputs = wfi.request['OutputDatasets'] 
    if ('EnableHarvesting' in wfi.request and not wfi.request['EnableHarvesting']) and ('DQMConfigCacheID' in wfi.request and wfi.request['DQMConfigCacheID']):
        if not 'MergedLFNBase' in wfi.request:
            print "f****d up"
            sendEmail('screwed up wl cache','%s wl cache is bad'%(wfi.request['RequestName']))
            all_OK['fake'] = False
            return all_OK,requests

        wfi = workflowInfo(url, wfi.request['RequestName'])
        dqms = [out for out in outputs if '/DQM' in out]
        if not all([in_full[dqm_input] for dqm_input in dqms]):
            wfi.sendLog('closor',"will not be able to assign the harvesting: holding up")
            for dqm_input in dqms:
                all_OK[dqm_input] = False
                ## raise the subscription to high priority
                sites = set(wfi.request['NonCustodialSites'])
                for site in sites:
                    res = updateSubscription(url, site, dqm_input, priority='reserved')
                    print "increased priority",res

                ## make a subscription somewhere else in the T1s at random each time, hoping it will get there eventually
                #pick = SI.CE_to_SE(random.choice(SI.sites_T1s))
                #print "replicating",dqm_input,"to",pick
                #res = makeReplicaRequest(url, pick, [dqm_input], "Replicating DQM for harvesting", priority='normal', approve=True, mail=False)

                return all_OK,requests

        for dqm_input in dqms:
            ## handle it properly
            harvesting_schema = {
                'Requestor': os.getenv('USER'),
                'RequestType' : 'DQMHarvest',
                'Group' : 'DATAOPS'
                }
            copy_over = [
                'AcquisitionEra',
                'ProcessingString',
                'DQMUploadUrl',
                'CMSSWVersion',
                'CouchDBName',
                'CouchWorkloadDBName',
                'ConfigCacheUrl',
                'DbsUrl',
                'inputMode',
                'DQMConfigCacheID',
                'OpenRunningTimeout',
                'ScramArch',
                'CMSSWVersion',
                'Campaign',
                'Memory', #dummy
                'SizePerEvent', #dummy
                'GlobalTag', #dummy
                ]
            for item in copy_over:
                if item in wfi.request:
                    harvesting_schema[item] = copy.deepcopy(wfi.request[item])
                else:
                    print item,"is not in initial schema"

            harvesting_schema['InputDataset'] = dqm_input
            harvesting_schema['TimePerEvent'] = 1
            harvesting_schema['PrepID'] = 'Harvest-'+wfi.request['PrepID']
            if len(wfi.request['RequestString'])>60:
                wfi.request['RequestString']= wfi.request['RequestString'][:60]
                print "truncating request string",wfi.request['RequestString']
                
            harvesting_schema['RequestString'] = 'HARVEST-'+wfi.request['RequestString']
            harvesting_schema['DQMHarvestUnit'] = 'byRun'
            harvesting_schema['RequestPriority'] = min(wfi.request['RequestPriority']*10,999999)

            harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema)
            if not harvest_request:
                print "Error in making harvesting for",wfi.request['RequestName']
                print "schema"
                print json.dumps( harvesting_schema, indent = 2)
                harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema)
                if not harvest_request:
                    print "Error twice in harvesting for",wfi.request['RequestName']
                    print "schema"
                    print json.dumps( harvesting_schema, indent = 2)

            if harvest_request:
                requests.append( harvest_request )
                ## should we protect for setting approved ? no, it's notified below, assignment will fail, likely
                data = reqMgrClient.setWorkflowApproved(url, harvest_request)
                print "created",harvest_request,"for harvesting of",dqm_input
                wfi.sendLog('closor',"created %s for harvesting of %s"%( harvest_request, dqm_input))
                ## assign it directly
                team = wfi.request['Team']
                parameters={
                    'SiteWhitelist' : [SI.SE_to_CE(se) for se in wfi.request['NonCustodialSites']],
                    'AcquisitionEra' : wfi.acquisitionEra(),
                    'ProcessingString' : wfi.processingString(),
                    'MergedLFNBase' : wfi.request['MergedLFNBase'], 
                    'ProcessingVersion' : wfi.request['ProcessingVersion'],
                    'execute' : True
                    }
                if in_full[dqm_input]:
                    print "using full copy at",in_full[dqm_input]
                    parameters['SiteWhitelist'] = [SI.SE_to_CE(se) for se in in_full[dqm_input]]
                else:
                    print "cannot do anything if not having a full copy somewhere"
                    
                    all_OK[dqm_input]=False
                    continue

                result = reqMgrClient.assignWorkflow(url, harvest_request, team, parameters)
                if not result:
                    #sendEmail('harvesting request created','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
                    wfi.sendLog('closor','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']))
                    sendLog('closor','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), level='critical')
                else:
                    #sendEmail('harvesting request assigned','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch']) 
                    wfi.sendLog('closor','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']))

            else:
                #print "could not make the harvesting for",wfo.name,"not announcing"
                wfi.sendLog('closor',"could not make the harvesting request")
                sendLog('closor',"could not make the harvesting request for %s"% wfi.request['RequestName'], level='critical')
                all_OK[dqm_input]=False                    
    return (all_OK, requests)
Beispiel #16
0
def rejector(url, specific, options=None):

    up = componentInfo()

    if specific and specific.startswith('/'):
        return

    if options.filelist:
        wfs = []
        for line in filter(None, open(options.filelist).read().split('\n')):
            print line
            wfs.extend(
                session.query(Workflow).filter(
                    Workflow.name.contains(line)).all())
    elif specific:
        wfs = session.query(Workflow).filter(
            Workflow.name.contains(specific)).all()
    else:
        return

    print len(wfs), "to reject"

    if len(wfs) > 1:
        print "\n".join([wfo.name for wfo in wfs])
        answer = raw_input('Reject these')
        if not answer.lower() in ['y', 'yes']:
            return

    for wfo in wfs:
        #wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject", spec
            return
        results = []
        wfi = workflowInfo(url, wfo.name)

        datasets = set(wfi.request['OutputDatasets'])
        reqMgrClient.invalidateWorkflow(
            url, wfo.name, current_status=wfi.request['RequestStatus'])

        wfi.sendLog('rejector',
                    'invalidating the workflow by unified operator')
        ## need to find the whole familly and reject the whole gang
        familly = getWorkflowById(url, wfi.request['PrepID'], details=True)
        for fwl in familly:
            if fwl['RequestDate'] < wfi.request['RequestDate']: continue
            if fwl['RequestType'] != 'Resubmission': continue
            if 'OriginalRequestName' in fwl and fwl[
                    'OriginalRequestName'] != wfi.request['RequestName']:
                continue
            print "rejecting", fwl['RequestName']
            reqMgrClient.invalidateWorkflow(
                url, fwl['RequestName'], current_status=fwl['RequestStatus'])
            datasets.update(fwl['OutputDatasets'])

        for dataset in datasets:
            if options.keep:
                print "keeping", dataset, "in its current status"
            else:
                results.append(setDatasetStatus(dataset, 'INVALID'))

        #if wfi.request['RequestStatus'] in ['rejected','rejected-archived','aborted','aborted-archived']:
        #    print 'already',wfi.request['RequestStatus']
        #    if not options.clone:
        #        wfo.status = 'forget'
        #        session.commit()
        #        continue

        if all(map(lambda result: result in ['None', None, True], results)):
            wfo.status = 'forget'
            session.commit()
            print wfo.name, "and", datasets, "are rejected"
            if options and options.clone:
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                schema['OriginalRequestName'] = wfo.name
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion'] = int(
                        schema['ProcessingVersion']
                    ) + 1  ## dubious str->int conversion
                else:
                    schema['ProcessingVersion'] = 2
                for k in schema.keys():
                    if k.startswith('Team'):
                        schema.pop(k)
                    if k.startswith('checkbox'):
                        schema.pop(k)

                ## a few tampering of the original request
                if options.Memory:
                    schema['Memory'] = options.Memory
                if options.deterministic:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['DeterministicPileup'] = True
                if options.EventsPerJob:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['EventsPerJob'] = options.EventsPerJob
                    else:
                        schema['EventsPerJob'] = options.EventsPerJob
                if options.EventAwareLumiBased:
                    schema['SplittingAlgo'] = 'EventAwareLumiBased'
                if options.TimePerEvent:
                    schema['TimePerEvent'] = options.TimePerEvent

                if options.ProcessingString:
                    schema['ProcessingString'] = options.ProcessingString
                if options.AcquisitionEra:
                    schema['AcquisitionEra'] = options.AcquisitionEra
                if options.runs:
                    schema['RunWhitelist'] = map(int, options.runs.split(','))
                if options.PrepID:
                    schema['PrepID'] = options.PrepID

                if schema['RequestType'] == 'TaskChain' and options.no_output:
                    ntask = schema['TaskChain']
                    for it in range(1, ntask - 1):
                        schema['Task%d' % it]['KeepOutput'] = False
                    schema['TaskChain'] = ntask - 1
                    schema.pop('Task%d' % ntask)

                ## update to the current priority
                schema['RequestPriority'] = wfi.request['RequestPriority']

                ## drop shit on the way to reqmgr2
                for p in [
                        'RequestStatus',
                        'RequestTransition',
                        'RequestorDN',
                        'RequestWorkflow',
                        'OutputDatasets',
                        'ReqMgr2Only',
                        #'Group',
                        'RequestDate',
                        #'ConfigCacheUrl',
                        'RequestName',
                        'timeStamp',
                        'SoftwareVersions',
                        'CouchURL'
                ]:
                    if p in schema:
                        schema.pop(p)
                        #pass
                print "submitting"
                print json.dumps(schema, indent=2)
                newWorkflow = reqMgrClient.submitWorkflow(url, schema)
                if not newWorkflow:
                    print "error in cloning", wfo.name
                    print json.dumps(schema, indent=2)
                    return
                print newWorkflow
                #m = re.search("details\/(.*)\'",response)
                #if not m:
                #    print "error in cloning",wfo.name
                #    print response
                #    print json.dumps( schema, indent=2 )
                #    return
                #newWorkflow = m.group(1)

                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfo.status = 'trouble'
                session.commit()
                wfi.sendLog(
                    'rejector',
                    'Cloned into %s by unified operator' % (newWorkflow))
        else:
            print "error in rejecting", wfo.name, results
Beispiel #17
0
def singleRecovery(url, task, initial, actions, do=False):
    print "Inside single recovery!"
    payload = {
        "Requestor" : os.getenv('USER'),
        "Group" : 'DATAOPS',
        "RequestType" : "Resubmission",
        "ACDCServer" : initial['CouchURL'],
        "ACDCDatabase" : "acdcserver",
        "OriginalRequestName" : initial['RequestName'],
        "OpenRunningTimeout" : 0
    }
    copy_over = ['PrepID','Campaign','RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ,'CMSSWVersion']
    for c in copy_over:
        if c in initial:
            payload[c] = copy.deepcopy(initial[c])
        else:
            print c,"not in the initial payload"

    #a massage ? boost the recovery over the initial wf
#    payload['RequestPriority'] *= 10
    #Max priority is 1M
    payload['RequestPriority'] = min(500000,  payload['RequestPriority']*2 ) ## never above 500k

    #change parameters based on actions here
    if actions:
        for action in actions:
            if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
                payload['Memory'] = actions[action]
                print "Memory set to " + actions[action]
                ## Taskchains needs to be treated special to set the memory to all tasks
                if 'TaskChain' in initial:
                    it = 1
                    while True:
                        t = 'Task%d'%it
                        it += 1
                        if t in initial:
                            payload[t] = copy.deepcopy(initial[t])
                            payload[t]['Memory'] = actions[action]
                        else:
                            break

            if action.startswith('split'):
                split_alert = (initial['RequestType'] in ['MonteCarlo'] )
                for key in initial:
                    if key == 'SplittingAlgo' and (initial[key] in ['EventBased']):
                        split_alert = True
                    elif key.startswith('Task') and key != 'TaskChain':
                        for key2 in initial[key]:
                            if key2 == 'TaskName':
                                print "task",task.split('/')[-1]
                                print "TaskName",initial[key][key2]
                                if (initial[key][key2] == task) and (initial[key][key2] in ['EventBased']):
                                    split_alert = True
                if split_alert:
                    sendLog('actor','Cannot change splitting for %s'%initial['RequestName'],level='warning')
                    print "I should not be doing splitting for this type of request",initial['RequestName']
                    return None

    acdc_round = 0
    initial_string = payload['RequestString']
    if initial_string.startswith('ACDC'):
        if initial_string[4].isdigit():
            acdc_round = int(initial_string[4])
        acdc_round += 1

    initial_string = initial_string.replace('ACDC_','').replace('ACDC%d_'%(acdc_round-1),'')
    payload['RequestString'] = 'ACDC%d_%s'%(acdc_round,initial_string)
    payload['InitialTaskPath'] = task

    if not do:
        print json.dumps( payload, indent=2)
        return None

    print "ACDC payload"
#    print json.dumps( payload , indent=2)
    print actions

    ## submit here
    acdc = reqMgrClient.submitWorkflow(url, payload)
    if not acdc:
        print "Error in making ACDC for",initial["RequestName"]
        acdc = reqMgrClient.submitWorkflow(url, payload)
        if not acdc:
            print "Error twice in making ACDC for",initial["RequestName"]
            sendLog('actor','Failed twice in making ACDCs for %s!'%initial['RequestName'],level='critical')                
            return None

    ## change splitting if requested
    if actions:
        for action in actions:
            if action.startswith('split'):
                acdcInfo = workflowInfo(url, acdc)
                splittings = acdcInfo.getSplittings()
                if actions[action] != 'Same' and actions[action] != 'max':
                    factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
                    for split in splittings:
                        for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                            if act in split:
                                print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
                                split[act] /= factor
                                print "to",split[act]
                                break
                        split['requestName'] = acdc
                        print "changing the splitting of",acdc
                        print json.dumps( split, indent=2 )
                        print reqMgrClient.setWorkflowSplitting(url, acdc, split )
                elif 'max' in actions[action]:
                    for split in splittings:
                        for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                            if act in split:
                                print "Changing %s (%d) "%( act, split[act]),
                                split[act] = 1
                                print "to max splitting ",split[act]
                                break
                        split['requestName'] = acdc
                        print "changing the splitting of",acdc
                        print json.dumps( split, indent=2 )
                        print reqMgrClient.setWorkflowSplitting(url, acdc, split )

    data = reqMgrClient.setWorkflowApproved(url, acdc)
    

    print data
    return acdc
Beispiel #18
0
def singleRecovery(url, task, initial, actions, do=False):
    payload = {
        "Requestor": os.getenv('USER'),
        "Group": 'DATAOPS',
        "RequestType": "Resubmission",
        "ACDCServer": initial['CouchURL'],
        "ACDCDatabase": "acdcserver",
        "OriginalRequestName": initial['RequestName']
    }
    copy_over = [
        'PrepID', 'RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group',
        'Memory', 'RequestString', 'CMSSWVersion'
    ]
    for c in copy_over:
        payload[c] = copy.deepcopy(initial[c])

    #a massage ? boost the recovery over the initial wf
    payload['RequestPriority'] *= 10

    if actions:
        for action in actions:
            #if action.startswith('split'):
            #    factor = int(action.split('-')[-1]) if '-' in action else 2
            #    print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
            #    ## mention it's taking 2 times longer to have a 2 times finer splitting
            #    payload['TimePerEvent'] = factor*payload['TimePerEvent']
            if action.startswith('mem'):
                increase = int(
                    action.split('-')[-1]) if '-' in action else 1000
                ## increase the memory requirement by 1G
                payload['Memory'] += increase
            if action.startswith('split') and (
                    initial['RequestType'] in ['MonteCarlo'] or
                (initial['RequestType'] in ['TaskChain']
                 and not 'InputDataset' in initial['Task1'])):
                print "I should not be doing splitting for this type of request", initial[
                    'RequestName']
                return None

    if payload['RequestString'].startswith('ACDC'):
        print "This is not allowed yet"
        return None
    payload['RequestString'] = 'ACDC_' + payload['RequestString']
    payload['InitialTaskPath'] = task

    if not do:
        print json.dumps(payload, indent=2)
        return None

    print json.dumps(payload, indent=2)

    ## submit
    acdc = reqMgrClient.submitWorkflow(url, payload)
    if not acdc:
        print "Error in making ACDC for", initial["RequestName"]
        acdc = reqMgrClient.submitWorkflow(url, payload)
        if not acdc:
            print "Error twice in making ACDC for", initial["RequestName"]
            return None

    ## perform modifications
    if actions:
        for action in actions:
            if action.startswith('split'):
                factor = int(action.split('-')[-1]) if '-' in action else 2
                acdcInfo = workflowInfo(url, acdc)
                splittings = acdcInfo.getSplittings()
                for split in splittings:
                    for act in [
                            'avg_events_per_job', 'events_per_job',
                            'lumis_per_job'
                    ]:
                        if act in split:
                            print "Changing %s (%d) by a factor %d" % (
                                act, split[act], factor),
                            split[act] /= factor
                            print "to", split[act]
                            break
                    split['requestName'] = acdc
                    print "changing the splitting of", acdc
                    print json.dumps(split, indent=2)
                    print reqMgrClient.setWorkflowSplitting(url, acdc, split)

    data = reqMgrClient.setWorkflowApproved(url, acdc)
    print data
    return acdc
Beispiel #19
0
def spawn_harvesting(url, wfi, sites_for_DQMHarvest):
    SI = global_SI()

    all_OK = {}
    requests = []
    outputs = wfi.request['OutputDatasets']
    if ('EnableHarvesting' in wfi.request
            and not wfi.request['EnableHarvesting']) and (
                'DQMConfigCacheID' in wfi.request
                and wfi.request['DQMConfigCacheID']):
        if not 'MergedLFNBase' in wfi.request:
            print "f****d up"
            sendEmail('screwed up wl cache',
                      '%s wl cache is bad' % (wfi.request['RequestName']))
            all_OK['fake'] = False
            return all_OK, requests

        wfi = workflowInfo(url, wfi.request['RequestName'])
        dqms = [out for out in outputs if '/DQM' in out]
        #if not all([in_full[dqm_input] for dqm_input in dqms]):
        #    wfi.sendLog('closor',"will not be able to assign the harvesting: holding up")
        #    for dqm_input in dqms:
        #        all_OK[dqm_input] = False
        ## raise the subscription to high priority

        for dqm_input in dqms:
            ## handle it properly
            harvesting_schema = {
                'Requestor': os.getenv('USER'),
                'RequestType': 'DQMHarvest',
                'Group': 'DATAOPS'
            }
            copy_over = [
                'AcquisitionEra',
                'ProcessingString',
                'DQMUploadUrl',
                'CMSSWVersion',
                'CouchDBName',
                'CouchWorkloadDBName',
                'ConfigCacheUrl',
                'DbsUrl',
                'inputMode',
                'DQMConfigCacheID',
                'OpenRunningTimeout',
                'ScramArch',
                'CMSSWVersion',
                'Campaign',
                'Memory',  #dummy
                'SizePerEvent',  #dummy
                'GlobalTag',  #dummy
            ]
            for item in copy_over:
                if item in wfi.request:
                    harvesting_schema[item] = copy.deepcopy(wfi.request[item])
                else:
                    print item, "is not in initial schema"

            harvesting_schema['InputDataset'] = dqm_input
            harvesting_schema['TimePerEvent'] = 1
            harvesting_schema['PrepID'] = 'Harvest-' + wfi.request['PrepID']
            if len(wfi.request['RequestString']) > 60:
                wfi.request['RequestString'] = wfi.request[
                    'RequestString'][:60]
                print "truncating request string", wfi.request['RequestString']

            harvesting_schema[
                'RequestString'] = 'HARVEST-' + wfi.request['RequestString']
            harvesting_schema['DQMHarvestUnit'] = 'byRun'
            harvesting_schema['RequestPriority'] = min(
                wfi.request['RequestPriority'] * 10, 999999)

            harvest_request = reqMgrClient.submitWorkflow(
                url, harvesting_schema)
            if not harvest_request:
                print "Error in making harvesting for", wfi.request[
                    'RequestName']
                print "schema"
                print json.dumps(harvesting_schema, indent=2)
                harvest_request = reqMgrClient.submitWorkflow(
                    url, harvesting_schema)
                if not harvest_request:
                    print "Error twice in harvesting for", wfi.request[
                        'RequestName']
                    print "schema"
                    print json.dumps(harvesting_schema, indent=2)

            if harvest_request:
                requests.append(harvest_request)
                ## should we protect for setting approved ? no, it's notified below, assignment will fail, likely
                data = reqMgrClient.setWorkflowApproved(url, harvest_request)
                print "created", harvest_request, "for harvesting of", dqm_input
                wfi.sendLog(
                    'closor', "created %s for harvesting of %s" %
                    (harvest_request, dqm_input))
                ## assign it directly
                team = wfi.request['Team']
                parameters = {
                    'SiteWhitelist': [
                        SI.SE_to_CE(se)
                        for se in wfi.request['NonCustodialSites']
                    ],
                    'AcquisitionEra':
                    wfi.acquisitionEra(),
                    'ProcessingString':
                    wfi.processingString(),
                    'MergedLFNBase':
                    wfi.request['MergedLFNBase'],
                    'ProcessingVersion':
                    wfi.request['ProcessingVersion'],
                    'execute':
                    True
                }
                #if in_full[dqm_input]:
                #    print "using full copy at",in_full[dqm_input]
                #    parameters['SiteWhitelist'] = [SI.SE_to_CE(se) for se in in_full[dqm_input]]
                #else:
                #    print "cannot do anything if not having a full copy somewhere"

                #    all_OK[dqm_input]=False
                #    continue
                parameters['SiteWhitelist'] = sites_for_DQMHarvest

                result = reqMgrClient.assignWorkflow(url, harvest_request,
                                                     team, parameters)
                if not result:
                    #sendEmail('harvesting request created','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
                    wfi.sendLog(
                        'closor',
                        '%s was created at announcement of %s in %s, failed to assign'
                        % (harvest_request, dqm_input,
                           wfi.request['RequestName']))
                    sendLog(
                        'closor',
                        '%s was created at announcement of %s in %s, failed to assign'
                        % (harvest_request, dqm_input,
                           wfi.request['RequestName']),
                        level='critical')
                else:
                    #sendEmail('harvesting request assigned','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
                    wfi.sendLog(
                        'closor',
                        '%s was created at announcement of %s in %s, and assigned'
                        % (harvest_request, dqm_input,
                           wfi.request['RequestName']))

            else:
                #print "could not make the harvesting for",wfo.name,"not announcing"
                wfi.sendLog('closor', "could not make the harvesting request")
                sendLog('closor',
                        "could not make the harvesting request for %s" %
                        wfi.request['RequestName'],
                        level='critical')
                all_OK[dqm_input] = False
    return (all_OK, requests)
Beispiel #20
0
def singleRecovery(url, task , initial, actions, do=False):
    payload = {
        "Requestor" : os.getenv('USER'),
        "Group" : 'DATAOPS',
        "RequestType" : "Resubmission",
        "ACDCServer" : initial['CouchURL'],
        "ACDCDatabase" : "acdcserver",
        "OriginalRequestName" : initial['RequestName']
        }
    copy_over = ['PrepID','RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ,'CMSSWVersion']        
    for c in copy_over:
        payload[c] = copy.deepcopy(initial[c])

    #a massage ? boost the recovery over the initial wf
    payload['RequestPriority'] *= 10

    if actions:
        for action in actions:
            #if action.startswith('split'):
            #    factor = int(action.split('-')[-1]) if '-' in action else 2
            #    print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
            #    ## mention it's taking 2 times longer to have a 2 times finer splitting
            #    payload['TimePerEvent'] = factor*payload['TimePerEvent']
            if action.startswith('mem'):
                increase = int(action.split('-')[-1]) if '-' in action else 1000
                ## increase the memory requirement by 1G
                payload['Memory'] += increase
            if action.startswith('split') and (initial['RequestType'] in ['MonteCarlo'] or (initial['RequestType'] in ['TaskChain'] and not 'InputDataset' in initial['Task1'])):
                print "I should not be doing splitting for this type of request",initial['RequestName']
                return None

    if payload['RequestString'].startswith('ACDC'):
        print "This is not allowed yet"
        return None
    payload['RequestString'] = 'ACDC_'+payload['RequestString']
    payload['InitialTaskPath'] = task 

    if not do:
        print json.dumps( payload, indent=2)
        return None

    print json.dumps( payload , indent=2)

    ## submit
    acdc = reqMgrClient.submitWorkflow(url, payload)
    if not acdc:
        print "Error in making ACDC for",initial["RequestName"]
        acdc = reqMgrClient.submitWorkflow(url, payload)
        if not acdc:
            print "Error twice in making ACDC for",initial["RequestName"]
            return None
    
    ## perform modifications
    if actions:
        for action in actions:
            if action.startswith('split'):
                factor = int(action.split('-')[-1]) if '-' in action else 2
                acdcInfo = workflowInfo(url, acdc)
                splittings = acdcInfo.getSplittings()
                for split in splittings:
                    for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                        if act in split:
                            print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
                            split[act] /= factor
                            print "to",split[act]
                            break
                    split['requestName'] = acdc
                    print "changing the splitting of",acdc
                    print json.dumps( split, indent=2 )
                    print reqMgrClient.setWorkflowSplitting(url, acdc, split )
                
    data = reqMgrClient.setWorkflowApproved(url, acdc)
    print data
    return acdc
Beispiel #21
0
def spawn_harvesting(url, wfi , in_full):
    SI = siteInfo()
    
    all_OK = {}
    requests = []
    outputs = wfi.request['OutputDatasets'] 
    if ('EnableHarvesting' in wfi.request and wfi.request['EnableHarvesting']) or ('DQMConfigCacheID' in wfi.request and wfi.request['DQMConfigCacheID']):
        if not 'MergedLFNBase' in wfi.request:
            print "f****d up"
            sendEmail('screwed up wl cache','%s wl cache is bad'%(wfi.request['RequestName']))
            all_OK['fake'] = False
            return all_OK,requests

        wfi = workflowInfo(url, wfi.request['RequestName'])
        dqms = [out for out in outputs if '/DQM' in out]
        if not all([in_full[dqm_input] for dqm_input in dqms]):
            wfi.sendLog('closor',"will not be able to assign the harvesting: holding up")
            for dqm_input in dqms:
                all_OK[dqm_input] = False
                return all_OK,requests

        for dqm_input in dqms:
            ## handle it properly
            harvesting_schema = {
                'Requestor': os.getenv('USER'),
                'RequestType' : 'DQMHarvest',
                'Group' : 'DATAOPS'
                }
            copy_over = ['ProcessingString',
                         'DQMUploadUrl',
                         'CMSSWVersion',
                         'CouchDBName',
                         'CouchWorkloadDBName',
                         'CouchURL',
                         'DbsUrl',
                         'inputMode',
                         'DQMConfigCacheID',
                         'OpenRunningTimeout',
                         'ScramArch',
                         'CMSSWVersion',
                         'Campaign',
                         'Memory', #dummy
                         'SizePerEvent', #dummy
                         'GlobalTag', #dummy
                         ]
            for item in copy_over:
                harvesting_schema[item] = copy.deepcopy(wfi.request[item])
            harvesting_schema['InputDataset'] = dqm_input
            harvesting_schema['TimePerEvent'] = 1
            harvesting_schema['PrepID'] = 'Harvest-'+wfi.request['PrepID']
            harvesting_schema['RequestString'] = 'HARVEST-'+wfi.request['RequestString']
            harvesting_schema['DQMHarvestUnit'] = 'byRun'
            harvesting_schema['ConfigCacheUrl'] = harvesting_schema['CouchURL'] ## uhm, how stupid is that ?
            harvesting_schema['RequestPriority'] = wfi.request['RequestPriority']*10

            harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema)
            if not harvest_request:
                print "Error in making harvesting for",wfo.name
                print "schema"
                print json.dumps( harvesting_schema, indent = 2)
                harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema)
                if not harvest_request:
                    print "Error twice in harvesting for",wfo.name
                    print "schema"
                    print json.dumps( harvesting_schema, indent = 2)

            if harvest_request:
                requests.append( harvest_request )
                ## should we protect for setting approved ? no, it's notified below, assignment will fail, likely
                data = reqMgrClient.setWorkflowApproved(url, harvest_request)
                print "created",harvest_request,"for harvesting of",dqm_input
                wfi.sendLog('closor',"created %s for harvesting of %s"%( harvest_request, dqm_input))
                ## assign it directly
                team = wfi.request['Teams'][0]
                parameters={
                    'SiteWhitelist' : [SI.SE_to_CE(se) for se in wfi.request['NonCustodialSites']],
                    'AcquisitionEra' : wfi.acquisitionEra(),
                    'ProcessingString' : wfi.processingString(),
                    'MergedLFNBase' : wfi.request['MergedLFNBase'], 
                    'ProcessingVersion' : wfi.request['ProcessingVersion'],
                    'execute' : True
                    }
                if in_full[dqm_input]:
                    print "using full copy at",in_full[dqm_input]
                    parameters['SiteWhitelist'] = [SI.SE_to_CE(se) for se in in_full[dqm_input]]
                else:
                    print "cannot do anything if not having a full copy somewhere"
                    all_OK[dqm_input]=False
                    continue

                result = reqMgrClient.assignWorkflow(url, harvest_request, team, parameters)
                if not result:
                    sendEmail('harvesting request created','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch'])
                else:
                    sendEmail('harvesting request assigned','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch']) 
                    wfi.sendLog('closor','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']))

            else:
                print "could not make the harvesting for",wfo.name,"not announcing"
                wfi.sendLog('closor',"could not make the harvesting request")
                all_OK[dqm_input]=False                    
    return (all_OK, requests)
Beispiel #22
0
def singleRecovery(url, task, initial, actions, do=False):
    print "Inside single recovery!"
    payload = {
        "Requestor": os.getenv('USER'),
        "Group": 'DATAOPS',
        "RequestType": "Resubmission",
        "ACDCServer": initial['ConfigCacheUrl'],
        "ACDCDatabase": "acdcserver",
        "OriginalRequestName": initial['RequestName'],
        "OpenRunningTimeout": 0
    }
    copy_over = [
        'PrepID', 'Campaign', 'RequestPriority', 'TimePerEvent',
        'SizePerEvent', 'Group', 'Memory', 'RequestString', 'CMSSWVersion'
    ]
    for c in copy_over:
        if c in initial:
            payload[c] = copy.deepcopy(initial[c])
        else:
            print c, "not in the initial payload"

    #a massage ? boost the recovery over the initial wf


#    payload['RequestPriority'] *= 10
#Max priority is 1M
    payload['RequestPriority'] = min(500000, payload['RequestPriority'] *
                                     2)  ## never above 500k

    #change parameters based on actions here
    if actions:
        for action in actions:
            if action.startswith('mem') and actions[action] != "" and actions[
                    action] != 'Same':
                #if multicore parameter is also used, need to scale memory by the new number of cores
                if 'multicore' in actions and actions['multicore'] != "":
                    continue
                ## Taskchains needs to be treated special to set the memory to all tasks
                set_to = int(actions[action])
                if 'TaskChain' in initial:
                    mem_dict = {}
                    it = 1
                    while True:
                        t = 'Task%d' % it
                        it += 1
                        if t in initial:
                            tname = payload.setdefault(t,
                                                       initial[t])['TaskName']
                            mem = mem_dict.setdefault(tname,
                                                      payload[t]['Memory'])
                            mem_dict[tname] = set_to
                        else:
                            break
                    payload['Memory'] = mem_dict
                    print "Memory set to: ", json.dumps(mem_dict, indent=2)
                else:
                    payload['Memory'] = set_to
                    print "Memory set to: ", set_to

            if action.startswith('multicore') and actions[action] != "":
                set_to = int(actions[action])
                ## Taskchains needs to be treated special to set the multicore and memory values to all tasks
                if 'TaskChain' in initial:
                    mem_dict = payload['Memory'] if type(
                        payload['Memory']) == dict else {}
                    core_dict = {}
                    it = 1
                    while True:
                        t = 'Task%d' % it
                        it += 1
                        if t in initial:
                            tname = payload.setdefault(t,
                                                       initial[t])['TaskName']
                            mem = mem_dict.setdefault(tname,
                                                      payload[t]['Memory'])

                            #Need to scale the memory by the new number of cores
                            initial_cores = payload[t].setdefault(
                                'Multicore', 1)

                            if 'memory' in actions and actions[
                                    'memory'] != "" and actions[
                                        'memory'] != 'Same':
                                mem = actions['memory']

                            fraction_constant = 0.4
                            mem_per_core_c = int((1 - fraction_constant) *
                                                 mem / float(initial_cores))

                            mem_dict[tname] = int(mem +
                                                  (set_to - initial_cores) *
                                                  mem_per_core_c)
                            core_dict[tname] = set_to

                            print "For ", t
                            print "Multicore set to ", set_to
                            print "Memory set to ", mem_dict[tname]
                        else:
                            break
                    payload['Memory'] = mem_dict
                    payload['Multicore'] = core_dict

                else:
                    #Need to scale the memory by the new number of cores
                    initial_cores = initial.setdefault('Multicore', 1)

                    mem = payload['Memory']
                    if 'memory' in actions and actions[
                            'memory'] != "" and actions['memory'] != 'Same':
                        mem = actions['memory']

                    fraction_constant = 0.4
                    mem_per_core_c = int(
                        (1 - fraction_constant) * mem / float(initial_cores))

                    payload['Multicore'] = set_to
                    payload['Memory'] = int(mem + (set_to - initial_cores) *
                                            mem_per_core_c)

                    print "Multicore set to ", set_to
                    print "Memory set to ", payload['Memory']

            if action.startswith('split'):

                split_alert = (initial['RequestType'] in ['MonteCarlo'])
                for key in initial:
                    if key == 'SplittingAlgo' and (initial[key]
                                                   in ['EventBased']):
                        split_alert = True
                    elif key.startswith('Task') and key != 'TaskChain':
                        for key2 in initial[key]:
                            if key2 == 'TaskName':
                                this_taskname = initial[key][key2]
                                recover_task = task.split('/')[-1]
                                print "For recovery of task", recover_task
                                print "Looking at task", this_taskname
                                if (recover_task == this_taskname) and (
                                        initial[key]['SplittingAlgo']
                                        in ['EventBased']):
                                    ## the task to be recovered is actually of the wrong type to allow change of splitting
                                    sendLog(
                                        'actor',
                                        'To recover on %s, changing the splitting on %s is not really allowed and this will be ignored instead of failing acdc.'
                                        %
                                        (task, initial[key]['SplittingAlgo']),
                                        level='critical')
                                    ## do not send an alert and stop the acdc
                                    #split_alert = True

                if split_alert:
                    sendLog('actor',
                            'Cannot change splitting for %s' %
                            initial['RequestName'],
                            level='critical')
                    print "I should not be doing splitting for this type of request", initial[
                        'RequestName']
                    return None

    acdc_round = 0
    initial_string = payload['RequestString']
    if initial_string.startswith('ACDC'):
        if initial_string[4].isdigit():
            acdc_round = int(initial_string[4])
        acdc_round += 1

    initial_string = initial_string.replace('ACDC_', '').replace(
        'ACDC%d_' % (acdc_round - 1), '')
    payload['RequestString'] = 'ACDC%d_%s' % (acdc_round, initial_string)
    payload['InitialTaskPath'] = task

    if not do:
        print json.dumps(payload, indent=2)
        return None

    print "ACDC payload"
    #    print json.dumps( payload , indent=2)
    print actions

    ## submit here
    acdc = reqMgrClient.submitWorkflow(url, payload)
    if not acdc:
        print "Error in making ACDC for", initial["RequestName"]
        acdc = reqMgrClient.submitWorkflow(url, payload)
        if not acdc:
            print "Error twice in making ACDC for", initial["RequestName"]
            sendLog('actor',
                    'Failed twice in making ACDCs for %s!' %
                    initial['RequestName'],
                    level='critical')
            return None

    ## change splitting if requested
    if actions:
        for action in actions:
            if action.startswith('split'):
                acdcInfo = workflowInfo(url, acdc)
                splittings = acdcInfo.getSplittingsNew(strip=True)
                if actions[action] != 'Same' and actions[action] != 'max':
                    factor = int(
                        actions[action][0:-1]) if 'x' in actions[action] else 2
                    for split in splittings:
                        split_par = split['splitParams']
                        if split['splitAlgo'] in ['EventBased']:
                            sendLog(
                                'actor',
                                "Changing the splitting on %s for %s is not permitted. Not changing."
                                % (split['splitAlgo'], initial["RequestName"]),
                                level='critical')
                            continue
                        for act in [
                                'avg_events_per_job', 'events_per_job',
                                'lumis_per_job'
                        ]:
                            if act in split_par:
                                print "Changing %s (%d) by a factor %d" % (
                                    act, split_par[act], factor),
                                split_par[act] /= factor
                                print "to", split_par[act]
                                break
                        #split['requestName'] = acdc
                        #print "changing the splitting of",acdc
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, acdc, split )

                elif 'max' in actions[action]:
                    for split in splittings:
                        split_par = split['splitParams']
                        for act in [
                                'avg_events_per_job', 'events_per_job',
                                'lumis_per_job'
                        ]:
                            if act in split_par:
                                print "Changing %s (%d) " % (act,
                                                             split_par[act]),
                                split_par[act] = 1
                                print "to max splitting ", split_par[act]
                                break
                        #split['requestName'] = acdc
                        #print "changing the splitting of",acdc
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, acdc, split )
                print "changing the splitting of", acdc
                print json.dumps(splittings, indent=2)
                done = reqMgrClient.setWorkflowSplitting(url, acdc, splittings)
                ## check on done == True

    data = reqMgrClient.setWorkflowApproved(url, acdc)

    print data
    return acdc
Beispiel #23
0
def singleClone(url, wfname, actions, comment, do=False):
    
    wfi = workflowInfo(url, wfname)
    payload = wfi.getSchema()
    initial = wfi.request

    payload['Requestor']           = os.getenv('USER')
    payload['Group']               = 'DATAOPS'
    payload['OriginalRequestName'] = initial['RequestName']
    payload['RequestPriority'] = initial['RequestPriority']

    if 'ProcessingVersion' in initial:
        payload['ProcessingVersion'] = int(initial['ProcessingVersion']) +1
    else:
        payload['ProcessingVersion'] = 2

        
    payload = reqMgrClient.purgeClonedSchema( payload )

    if actions:
        for action in actions:
            if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
                if 'TaskChain' in payload:
                    print "Setting memory for clone of task chain"
                    mem_dict = {}
                    it=1
                    while True:
                        t = 'Task%d'%it
                        it+=1
                        if t in payload:
                            tname = payload[t]['TaskName']
                            mem_dict[tname] = int(actions[action])
                            print "Memory set for Task%d"%it
                        else:
                            break
                    payload['Memory'] = mem_dict
                else:
                    print "Setting memory for non-taskchain workflow"
                    payload['Memory'] = int(actions[action])
                print "Memory set to " + actions[action]

    print "Clone payload"
#    print json.dumps( payload , indent=2)
    print actions

    #Create clone
    clone = reqMgrClient.submitWorkflow(url, payload)
    if not clone:
        print "Error in making clone for",initial["RequestName"]
        clone = reqMgrClient.submitWorkflow(url, payload)
        if not clone:
            print "Error twice in making clone for",initial["RequestName"]
            sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"],level='critical')
            wfi.sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"])
            return None

    if actions:
        for action in actions:
            if action.startswith('split'):
                cloneinfo = workflowInfo(url, clone)
                splittings = cloneinfo.getSplittingsNew(strip=True)
                if actions[action] != 'Same' and actions[action] != 'max' and actions[action] != '':
                    factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
                    for split in splittings:
                        split_par = split['splitParams']
                        for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                            if act in split_par:
                                wfi.sendLog('actor','Changing %s (%d) by a factor %d'%( act, split_par[act], factor))
                                split_par[act] /= factor
                                print "to",split_par[act]
                                break
                        #split['requestName'] = clone
                        #print "changing the splitting of",clone
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, clone, split )
                elif 'max' in actions[action]:
                    for split in splittings:
                        split_par = split['splitParams']
                        for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                            if act in split_par:
                                wfi.sendLog('actor','Max splitting set for %s (%d'%( act, split_par[act]))
                                print "Changing %s (%d) "%( act, split_par[act]),
                                split_par[act] = 1
                                print "to max splitting ",split_par[act]
                                break
                        #split['requestName'] = clone
                        #print "changing the splitting of",clone
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, clone, split )

                print "changing the splitting of",clone
                print json.dumps( splittings, indent=2 )
                print reqMgrClient.setWorkflowSplitting(url, clone, splittings )
    #Approve
    data = reqMgrClient.setWorkflowApproved(url, clone)
    #wfi.sendLog('actor','Cloned into %s'%clone)

    
#    wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
#    wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)

    print data
    return clone
Beispiel #24
0
def singleClone(url, wfname, actions, comment, do=False):

    wfi = workflowInfo(url, wfname)
    payload = wfi.getSchema()
    initial = wfi.request

    payload['Requestor'] = os.getenv('USER')
    payload['Group'] = 'DATAOPS'
    payload['OriginalRequestName'] = initial['RequestName']
    payload['RequestPriority'] = initial['RequestPriority']

    if 'ProcessingVersion' in initial:
        payload['ProcessingVersion'] = int(initial['ProcessingVersion']) + 1
    else:
        payload['ProcessingVersion'] = 2


## drop parameters on the way to reqmgr2
    paramBlacklist = [
        'BlockCloseMaxEvents', 'BlockCloseMaxFiles', 'BlockCloseMaxSize',
        'BlockCloseMaxWaitTime', 'CouchWorkloadDBName', 'CustodialGroup',
        'CustodialSubType', 'Dashboard', 'GracePeriod', 'HardTimeout',
        'InitialPriority', 'inputMode', 'MaxMergeEvents', 'MaxMergeSize',
        'MaxRSS', 'MaxVSize', 'MinMergeSize', 'NonCustodialGroup',
        'NonCustodialSubType', 'OutputDatasets', 'ReqMgr2Only', 'RequestDate'
        'RequestorDN', 'RequestName', 'RequestStatus', 'RequestTransition',
        'RequestWorkflow', 'SiteWhitelist', 'SoftTimeout', 'SoftwareVersions',
        'SubscriptionPriority', 'Team', 'timeStamp', 'TrustSitelists',
        'TrustPUSitelists', 'TotalEstimatedJobs', 'TotalInputEvents',
        'TotalInputLumis', 'TotalInputFiles', 'checkbox', 'DN',
        'AutoApproveSubscriptionSites', 'NonCustodialSites', 'CustodialSites',
        'OriginalRequestName', 'Teams', 'OutputModulesLFNBases',
        'SiteBlacklist', 'AllowOpportunistic', '_id', 'Override'
    ]
    for p in paramBlacklist:
        if p in payload:
            payload.pop(p)

    taskParamBlacklist = ['EventsPerJob']
    for i in range(1, 100):
        t = 'Task%s' % i
        if not t in payload: break
        for p in taskParamBlacklist:
            if p in payload[t]:
                payload[t].pop(p)

    if actions:
        for action in actions:
            if action.startswith('mem') and actions[action] != "" and actions[
                    action] != 'Same':
                if 'TaskChain' in payload:
                    print "Setting memory for clone of task chain"
                    mem_dict = {}
                    it = 1
                    while True:
                        t = 'Task%d' % it
                        it += 1
                        if t in payload:
                            tname = payload[t]['TaskName']
                            mem_dict[tname] = int(actions[action])
                            print "Memory set for Task%d" % it
                        else:
                            break
                    payload['Memory'] = mem_dict
                else:
                    print "Setting memory for non-taskchain workflow"
                    payload['Memory'] = int(actions[action])
                print "Memory set to " + actions[action]

    print "Clone payload"
    #    print json.dumps( payload , indent=2)
    print actions

    #Create clone
    clone = reqMgrClient.submitWorkflow(url, payload)
    if not clone:
        print "Error in making clone for", initial["RequestName"]
        clone = reqMgrClient.submitWorkflow(url, payload)
        if not clone:
            print "Error twice in making clone for", initial["RequestName"]
            sendLog('actor',
                    'Failed to make a clone twice for %s!' %
                    initial["RequestName"],
                    level='critical')
            wfi.sendLog(
                'actor', 'Failed to make a clone twice for %s!' %
                initial["RequestName"])
            return None

    if actions:
        for action in actions:
            if action.startswith('split'):
                cloneinfo = workflowInfo(url, clone)
                splittings = cloneinfo.getSplittingsNew(strip=True)
                if actions[action] != 'Same' and actions[
                        action] != 'max' and actions[action] != '':
                    factor = int(
                        actions[action][0:-1]) if 'x' in actions[action] else 2
                    for split in splittings:
                        split_par = split['splitParams']
                        for act in [
                                'avg_events_per_job', 'events_per_job',
                                'lumis_per_job'
                        ]:
                            if act in split_par:
                                wfi.sendLog(
                                    'actor',
                                    'Changing %s (%d) by a factor %d' %
                                    (act, split_par[act], factor))
                                split_par[act] /= factor
                                print "to", split_par[act]
                                break
                        #split['requestName'] = clone
                        #print "changing the splitting of",clone
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, clone, split )
                elif 'max' in actions[action]:
                    for split in splittings:
                        split_par = split['splitParams']
                        for act in [
                                'avg_events_per_job', 'events_per_job',
                                'lumis_per_job'
                        ]:
                            if act in split_par:
                                wfi.sendLog(
                                    'actor', 'Max splitting set for %s (%d' %
                                    (act, split_par[act]))
                                print "Changing %s (%d) " % (act,
                                                             split_par[act]),
                                split_par[act] = 1
                                print "to max splitting ", split_par[act]
                                break
                        #split['requestName'] = clone
                        #print "changing the splitting of",clone
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, clone, split )

                print "changing the splitting of", clone
                print json.dumps(splittings, indent=2)
                print reqMgrClient.setWorkflowSplitting(url, clone, splittings)
    #Approve
    data = reqMgrClient.setWorkflowApproved(url, clone)
    wfi.sendLog('actor', 'Cloned into %s' % clone)

    #    wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
    #    wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)

    print data
    return clone
Beispiel #25
0
def singleRecovery(url, task , initial, actions, do=False):
    payload = {
        "Requestor" : os.getenv('USER'),
        "Group" : 'DATAOPS',
        "RequestType" : "Resubmission",
        "ACDCServer" : initial['ConfigCacheUrl'],
        "ACDCDatabase" : "acdcserver",
        "OriginalRequestName" : initial['RequestName'],
        "OpenRunningTimeout" : 0
        }
    copy_over = ['PrepID', 'Campaign', 'RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ,'CMSSWVersion']        
    for c in copy_over:
        if c in initial:
            payload[c] = copy.deepcopy(initial[c])
        else:
            print c,"not in the initial payload"

    #a massage ? boost the recovery over the initial wf
    payload['RequestPriority'] *= 2
    payload['RequestPriority'] = min(500000, payload['RequestPriority'])

    if actions:
        for action in actions:
            #if action.startswith('split'):
            #    factor = int(action.split('-')[-1]) if '-' in action else 2
            #    print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
            #    ## mention it's taking 2 times longer to have a 2 times finer splitting
            #    payload['TimePerEvent'] = factor*payload['TimePerEvent']
            if action.startswith('mem'):
                arg = action.split('-',1)[-1]
                increase = set_to = None
                tasks,set_to = arg.split(':') if ':' in arg else (None,arg)
                tasks = tasks.split(',') if tasks else []
                if set_to.startswith('+'):
                    increase = int(set_to[1:])
                else:
                    set_to = int(set_to)
                ## increase the memory requirement by 1G

                if 'TaskChain' in initial:
                    mem_dict = {} 
                    it = 1
                    while True:
                        t = 'Task%d'%it
                        it += 1
                        if t in initial:
                            tname = payload.setdefault(t, initial[t])['TaskName']
                            mem = mem_dict.setdefault( tname, payload[t]['Memory'])
                            if tasks and not tname in tasks:
                                print tname,"not concerned"
                                continue
                            if set_to:
                                mem_dict[tname] = set_to
                            else:
                                mem_dict[tname] += increase
                        else:
                            break
                    payload['Memory'] = mem_dict
                else:
                    payload['Memory'] = set_to
                #increase = int(action.split('-')[-1]) if '-' in action else 1000
                ## increase the memory requirement by 1G
                #payload['Memory'] += increase

            if action.startswith('split') and (initial['RequestType'] in ['MonteCarlo'] or (initial['RequestType'] in ['TaskChain'] and not 'InputDataset' in initial['Task1'])):
                print "I should not be doing splitting for this type of request",initial['RequestName']
                return None
            if action.startswith('core'):
                arg = action.split('-',1)[-1]
                tasks,set_to = arg.split(':') if ':' in arg else (None,arg)
                tasks = tasks.split(',') if tasks else []
                set_to = int(set_to)
                if 'TaskChain' in initial:
                    core_dict = {}
                    mem_dict = payload['Memory'] if type(payload['Memory'])==dict else {}
                    it = 1
                    while True:
                        t = 'Task%d'%it
                        it += 1
                        if t in initial:
                            tname = payload.setdefault(t, initial[t])['TaskName']
                            mcore = core_dict.setdefault(tname, payload[t]['Multicore'])
                            mem = mem_dict.setdefault(tname, payload[t]['Memory'])
                            if tasks and not tname in tasks:
                                print tname,"not concerned"
                                continue

                            factor = (set_to / float(mcore))
                            fraction_constant = 0.4 
                            mem_per_core_c = int((1-fraction_constant) * mem / float(mcore))
                            ##scale the memory 
                            mem_dict[tname] += (set_to-mcore)*mem_per_core_c
                            ## scale time/event
                            time_dict[tname] = payload[t]['TimePerEvent'] /factor
                            ## set the number of cores
                            core_dict[tname] = set_to
                        else: 
                            break
                    payload['Multicore'] = core_dict
                    ##payload['TimePerEvent'] = time_dict ## cannot be used yet
                else:
                    payload['Multicore'] = increase

    acdc_round = 0
    initial_string = payload['RequestString']
    if initial_string.startswith('ACDC'):
        if initial_string[4].isdigit():
            acdc_round = int(initial_string[4])
        acdc_round += 1
        #print acdc_round
        #print "This is not allowed yet"
        #return None
    initial_string = initial_string.replace('ACDC_','').replace('ACDC%d_'%(acdc_round-1),'')
    payload['RequestString'] = 'ACDC%d_%s'%(acdc_round,initial_string)
    payload['InitialTaskPath'] = task 

    if not do:
        print json.dumps( payload, indent=2)
        return None

    print "ACDC payload"
    print json.dumps( payload , indent=2)
    print actions

    ## submit
    acdc = reqMgrClient.submitWorkflow(url, payload)
    if not acdc:
        print "Error in making ACDC for",initial["RequestName"]
        acdc = reqMgrClient.submitWorkflow(url, payload)
        if not acdc:
            print "Error twice in making ACDC for",initial["RequestName"]
            return None
    
    ## perform modifications
    if actions:
        for action in actions:
            if action.startswith('split'):
                factor = int(action.split('-')[-1]) if '-' in action else 2
                acdcInfo = workflowInfo(url, acdc)
                splittings = acdcInfo.getSplittings()
                for split in splittings:
                    for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                        if act in split:
                            print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
                            split[act] /= factor
                            print "to",split[act]
                            break
                    split['requestName'] = acdc
                    print "changing the splitting of",acdc
                    print json.dumps( split, indent=2 )
                    print reqMgrClient.setWorkflowSplitting(url, acdc, split )
                
    data = reqMgrClient.setWorkflowApproved(url, acdc)
    print data
    return acdc
 def testSetWorkflowApproved(self):
     # setWorkflowApproved
     te = reqMgr.setWorkflowApproved(url, testwf)
     if te is None:
         self.fail("None output")
Beispiel #27
0
def rejector(url, specific, options=None):

    
    up = componentInfo(soft=['wtc','jira'])
    if not up.check(): return

    if specific and specific.startswith('/'):
        ## this is for a dataset
        print setDatasetStatus(specific, 'INVALID')
        return

    if options.filelist:
        wfs = []
        for line in filter(None, open(options.filelist).read().split('\n')):
            print line
            wfs.extend( session.query(Workflow).filter(Workflow.name.contains(line)).all())
    elif specific:
        wfs = session.query(Workflow).filter(Workflow.name.contains(specific)).all()
        if not wfs:
            batches = batchInfo().content()
            for bname in batches:
                if specific == bname:
                    for pid in batches[bname]:
                        b_wfs = getWorkflowById(url, pid)
                        for wf in b_wfs:
                            wfs.append( session.query(Workflow).filter(Workflow.name == wf).first())
                    break
    else:
        wfs = session.query(Workflow).filter(Workflow.status == 'assistance-clone').all()
        #wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
        ## be careful then on clone case by case
        options.clone = True
        print "not supposed to function yet"
        return 

    print len(wfs),"to reject"

    if len(wfs)>1:
        print "\n".join( [wfo.name for wfo in wfs] )
        answer = raw_input('Reject these')
        if not answer.lower() in ['y','yes']:
            return
        
    for wfo in wfs:
        #wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject",spec
            return
        wfi = workflowInfo(url, wfo.name)

        comment=""
        if options.comments: comment = ", reason: "+options.comments
        if options.keep: 
            wfi.sendLog('rejector','invalidating the workflow by unified operator%s'%comment)
        else:
            wfi.sendLog('rejector','invalidating the workflow and outputs by unified operator%s'%comment)

        results = invalidate(url, wfi, only_resub=True, with_output= (not options.keep))

        if all(results):
            print wfo.name,"rejected"
            if options and options.clone:
                wfo.status = 'trouble'
                session.commit()                
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                schema['OriginalRequestName'] = wfo.name
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion'] = int(schema['ProcessingVersion'])+1 ## dubious str->int conversion
                else:
                    schema['ProcessingVersion']=2
                for k in schema.keys():
                    if k.startswith('Team'):
                        schema.pop(k)
                    if k.startswith('checkbox'):
                        schema.pop(k)

                ## a few tampering of the original request
                if options.Memory:
                    if schema['RequestType'] == 'TaskChain':
                        it=1
                        while True:
                            t = 'Task%d'%it
                            it+=1
                            if t in schema:
                                schema[t]['Memory'] = options.Memory
                            else:
                                break
                    else:
                        schema['Memory'] = options.Memory
                        
                if options.Multicore:
                    ## to do : set it properly in taskchains
                    if schema['RequestType'] == 'TaskChain':
                        tasks,set_to = options.Multicore.split(':') if ':' in options.Multicore else ("",options.Multicore)
                        set_to = int(set_to)
                        tasks = tasks.split(',') if tasks else ['Task1']
                        it = 1 
                        while True:
                            tt = 'Task%d'% it
                            it+=1
                            if tt in schema:
                                tname = schema[tt]['TaskName']
                                if tname in tasks or tt in tasks:
                                    mem = schema[tt]['Memory']
                                    mcore = schema[tt].get('Multicore',1)
                                    factor = (set_to / float(mcore))
                                    fraction_constant = 0.4
                                    mem_per_core_c = int((1-fraction_constant) * mem / float(mcore))
                                    print "mem per core", mem_per_core_c
                                    print "base mem", mem
                                    ## adjusting the parameter in the clone
                                    schema[tt]['Memory'] += (set_to-mcore)*mem_per_core_c
                                    schema[tt]['Multicore'] = set_to
                                    schema[tt]['TimePerEvent'] /= factor
                            else:
                                break
                    else:
                        schema['Multicore'] = options.Multicore
                if options.deterministic:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['DeterministicPileup']  = True
                if options.EventsPerJob:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['EventsPerJob'] = options.EventsPerJob
                    else:
                        schema['EventsPerJob'] = options.EventsPerJob
                if options.EventAwareLumiBased:
                    schema['SplittingAlgo'] = 'EventAwareLumiBased'
                if options.TimePerEvent:
                    schema['TimePerEvent'] = options.TimePerEvent

                if options.ProcessingString:
                    schema['ProcessingString'] = options.ProcessingString
                if options.AcquisitionEra:
                    schema['AcquisitionEra'] = options.AcquisitionEra
                if options.runs:
                    schema['RunWhitelist'] = map(int,options.runs.split(','))
                if options.PrepID:
                    schema['PrepID'] =options.PrepID

                if schema['RequestType'] == 'TaskChain' and options.no_output:
                    ntask = schema['TaskChain']
                    for it in range(1,ntask-1):
                        schema['Task%d'%it]['KeepOutput'] = False
                    schema['TaskChain'] = ntask-1
                    schema.pop('Task%d'%ntask)

                if options.priority:
                    schema['RequestPriority'] = options.priority

                ## update to the current priority
                schema['RequestPriority'] = wfi.request['RequestPriority']

                ## drop shit on the way to reqmgr2
                schema = reqMgrClient.purgeClonedSchema( schema )

                print "submitting"
                if (options.to_stepchain and (schema['RequestType']=='TaskChain')):
                    ## transform the schema into StepChain schema
                    print "Transforming a TaskChain into a StepChain"
                    mcore = 0
                    mem = 0
                    schema['RequestType'] = 'StepChain'
                    schema['StepChain'] = schema.pop('TaskChain')
                    schema['SizePerEvent'] = 0
                    schema['TimePerEvent'] = 0
                    step=1
                    s_n = {}
                    while True:
                        if 'Task%d'%step in schema:
                            sname = 'Step%d'%step
                            schema[sname] = schema.pop('Task%d'%step)
                            tmcore = schema[sname].pop('Multicore')
                            tmem = schema[sname].pop('Memory')
                            if mcore and tmcore != mcore:
                                wfi.sendLog('rejector','the conversion to stepchain encoutered different value of Multicore %d != %d'%( tmcore, mcore))
                                sendLog('rejector','the conversion of %s to stepchain encoutered different value of Multicore %d != %d'%( wfo.name, tmcore, mcore), level='critical')
                            mcore = max(mcore, tmcore)
                            mem = max(mem, tmem)
                            schema[sname]['StepName'] = schema[sname].pop('TaskName')
                            s_n[ schema[sname]['StepName'] ] = sname
                            if 'InputTask' in schema[sname]:
                                schema[sname]['InputStep'] = schema[sname].pop('InputTask')
                            eff = 1.
                            up_s = sname
                            while True:
                                ## climb up a step. supposedely already all converted
                                up_s = s_n.get(schema[up_s].get('InputStep',None),None)
                                if up_s:
                                    ## multiply with the efficiency
                                    eff *= schema[up_s].get('FilterEfficiency',1.)
                                else:
                                    ## or stop there
                                    break

                            if not 'KeepOutput' in schema[sname]:
                                ## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
                                schema[sname]['KeepOutput'] = False
                            schema['TimePerEvent'] += eff*schema[sname].pop('TimePerEvent')
                            schema['SizePerEvent'] += eff*schema[sname].pop('SizePerEvent')
                            step+=1
                        else:
                            break
                    schema['Multicore'] = mcore
                    schema['Memory'] = mem
                print json.dumps( schema, indent=2 )
                newWorkflow = reqMgrClient.submitWorkflow(url, schema)
                if not newWorkflow:
                    msg = "Error in cloning {}".format(wfo.name)
                    print(msg)
                    wfi.sendLog('rejector',msg)
                          
                    # Get the error message
                    time.sleep(5)
                    data = reqMgrClient.requestManagerPost(url, "/reqmgr2/data/request", schema)
                    wfi.sendLog('rejector',data)
                    
                    print json.dumps( schema, indent=2 )
                    return 
                print newWorkflow

                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfi.sendLog('rejector','Cloned into %s by unified operator %s'%( newWorkflow, comment ))
                wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False)
            else:
                wfo.status = 'trouble' if options.set_trouble else 'forget' 
                wfi.notifyRequestor('Rejected by unified operator %s'%( comment ),do_batch=False)
                session.commit()

        else:
            msg = "Error in rejecting {}: {}".format(wfo.name,results)
            print(msg)
            wfi.sendLog('rejector',msg)
 def testSetWorkflowApproved(self):
     # setWorkflowApproved
     te = reqMgr.setWorkflowApproved(url, testwf)
     if te is None:
         self.fail("None output")
Beispiel #29
0
def rejector(url, specific, options=None):

    up = componentInfo(soft=['wtc'])
    if not up.check(): return

    if specific and specific.startswith('/'):
        ## this is for a dataset
        print setDatasetStatus(specific, 'INVALID')
        return

    if options.filelist:
        wfs = []
        for line in filter(None, open(options.filelist).read().split('\n')):
            print line
            wfs.extend(
                session.query(Workflow).filter(
                    Workflow.name.contains(line)).all())
    elif specific:
        wfs = session.query(Workflow).filter(
            Workflow.name.contains(specific)).all()
        if not wfs:
            batches = json.loads(open('batches.json').read())
            for bname in batches:
                if specific == bname:
                    for wf in batches[bname]:
                        wfs.append(
                            session.query(Workflow).filter(
                                Workflow.name == wf).first())
    else:
        wfs = session.query(Workflow).filter(
            Workflow.status == 'assistance-clone').all()
        #wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
        ## be careful then on clone case by case
        options.clone = True
        print "not supposed to function yet"
        return

    print len(wfs), "to reject"

    if len(wfs) > 1:
        print "\n".join([wfo.name for wfo in wfs])
        answer = raw_input('Reject these')
        if not answer.lower() in ['y', 'yes']:
            return

    for wfo in wfs:
        #wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject", spec
            return
        results = []
        wfi = workflowInfo(url, wfo.name)

        datasets = set(wfi.request['OutputDatasets'])
        reqMgrClient.invalidateWorkflow(
            url, wfo.name, current_status=wfi.request['RequestStatus'])

        comment = ""
        if options.comments: comment = ", reason: " + options.comments
        wfi.sendLog(
            'rejector',
            'invalidating the workflow by unified operator%s' % comment)
        ## need to find the whole familly and reject the whole gang
        familly = getWorkflowById(url, wfi.request['PrepID'], details=True)
        for fwl in familly:
            if fwl['RequestDate'] < wfi.request['RequestDate']: continue
            if fwl['RequestType'] != 'Resubmission': continue
            ## does not work on second order acd
            #if 'OriginalRequestName' in fwl and fwl['OriginalRequestName'] != wfi.request['RequestName']: continue
            print "rejecting", fwl['RequestName']
            reqMgrClient.invalidateWorkflow(
                url,
                fwl['RequestName'],
                current_status=fwl['RequestStatus'],
                cascade=False)
            datasets.update(fwl['OutputDatasets'])

        for dataset in datasets:
            if options.keep:
                print "keeping", dataset, "in its current status"
            else:
                results.append(setDatasetStatus(dataset, 'INVALID'))
                pass

        if all(map(lambda result: result in ['None', None, True], results)):
            print wfo.name, "and", datasets, "are rejected"
            if options and options.clone:
                wfo.status = 'trouble'
                session.commit()
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                schema['OriginalRequestName'] = wfo.name
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion'] = int(
                        schema['ProcessingVersion']
                    ) + 1  ## dubious str->int conversion
                else:
                    schema['ProcessingVersion'] = 2
                for k in schema.keys():
                    if k.startswith('Team'):
                        schema.pop(k)
                    if k.startswith('checkbox'):
                        schema.pop(k)

                ## a few tampering of the original request
                if options.Memory:
                    if schema['RequestType'] == 'TaskChain':
                        it = 1
                        while True:
                            t = 'Task%d' % it
                            it += 1
                            if t in schema:
                                schema[t]['Memory'] = options.Memory
                            else:
                                break
                    else:
                        schema['Memory'] = options.Memory

                if options.Multicore:
                    ## to do : set it properly in taskchains
                    if schema['RequestType'] == 'TaskChain':
                        tasks, set_to = options.Multicore.split(
                            ':') if ':' in options.Multicore else (
                                "", options.Multicore)
                        set_to = int(set_to)
                        tasks = tasks.split(',') if tasks else ['Task1']
                        it = 1
                        while True:
                            tt = 'Task%d' % it
                            it += 1
                            if tt in schema:
                                tname = schema[tt]['TaskName']
                                if tname in tasks or tt in tasks:
                                    mem = schema[tt]['Memory']
                                    mcore = schema[tt].get('Multicore', 1)
                                    factor = (set_to / float(mcore))
                                    fraction_constant = 0.4
                                    mem_per_core_c = int(
                                        (1 - fraction_constant) * mem /
                                        float(mcore))
                                    print "mem per core", mem_per_core_c
                                    print "base mem", mem
                                    ## adjusting the parameter in the clone
                                    schema[tt]['Memory'] += (
                                        set_to - mcore) * mem_per_core_c
                                    schema[tt]['Multicore'] = set_to
                                    schema[tt]['TimePerEvent'] /= factor
                            else:
                                break
                    else:
                        schema['Multicore'] = options.Multicore
                if options.deterministic:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['DeterministicPileup'] = True
                if options.EventsPerJob:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['EventsPerJob'] = options.EventsPerJob
                    else:
                        schema['EventsPerJob'] = options.EventsPerJob
                if options.EventAwareLumiBased:
                    schema['SplittingAlgo'] = 'EventAwareLumiBased'
                if options.TimePerEvent:
                    schema['TimePerEvent'] = options.TimePerEvent

                if options.ProcessingString:
                    schema['ProcessingString'] = options.ProcessingString
                if options.AcquisitionEra:
                    schema['AcquisitionEra'] = options.AcquisitionEra
                if options.runs:
                    schema['RunWhitelist'] = map(int, options.runs.split(','))
                if options.PrepID:
                    schema['PrepID'] = options.PrepID

                if schema['RequestType'] == 'TaskChain' and options.no_output:
                    ntask = schema['TaskChain']
                    for it in range(1, ntask - 1):
                        schema['Task%d' % it]['KeepOutput'] = False
                    schema['TaskChain'] = ntask - 1
                    schema.pop('Task%d' % ntask)

                if options.priority:
                    schema['RequestPriority'] = options.priority

                ## update to the current priority
                schema['RequestPriority'] = wfi.request['RequestPriority']

                ## drop shit on the way to reqmgr2
                schema = reqMgrClient.purgeClonedSchema(schema)

                print "submitting"
                if (options.to_stepchain
                        and (schema['RequestType'] == 'TaskChain')):
                    ## transform the schema into StepChain schema
                    print "Transforming a TaskChain into a StepChain"
                    schema['RequestType'] = 'StepChain'
                    schema['StepChain'] = schema.pop('TaskChain')
                    schema['SizePerEvent'] = 0
                    schema['TimePerEvent'] = 0
                    step = 1
                    s_n = {}
                    while True:
                        if 'Task%d' % step in schema:
                            schema['Step%d' % step] = schema.pop('Task%d' %
                                                                 step)
                            schema['Step%d' % step]['StepName'] = schema[
                                'Step%d' % step].pop('TaskName')
                            s_n[schema['Step%d' %
                                       step]['StepName']] = 'Step%d' % step
                            if 'InputTask' in schema['Step%d' % step]:
                                schema['Step%d' % step]['InputStep'] = schema[
                                    'Step%d' % step].pop('InputTask')
                            eff = 1.
                            up_s = 'Step%d' % step
                            while True:
                                ## climb up a step. supposedely already all converted
                                up_s = s_n.get(
                                    schema[up_s].get('InputStep', None), None)
                                if up_s:
                                    ## multiply with the efficiency
                                    eff *= schema[up_s].get(
                                        'FilterEfficiency', 1.)
                                else:
                                    ## or stop there
                                    break

                            if not 'KeepOutput' in schema['Step%d' % step]:
                                ## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
                                schema['Step%d' % step]['KeepOutput'] = False
                            schema['TimePerEvent'] += eff * schema[
                                'Step%d' % step].pop('TimePerEvent')
                            schema['SizePerEvent'] += eff * schema[
                                'Step%d' % step].pop('SizePerEvent')
                            step += 1
                        else:
                            break

                print json.dumps(schema, indent=2)
                newWorkflow = reqMgrClient.submitWorkflow(url, schema)
                if not newWorkflow:
                    print "error in cloning", wfo.name
                    print json.dumps(schema, indent=2)
                    return
                print newWorkflow

                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfi.sendLog(
                    'rejector', 'Cloned into %s by unified operator %s' %
                    (newWorkflow, comment))
                wfi.notifyRequestor('Cloned into %s by unified operator %s' %
                                    (newWorkflow, comment),
                                    do_batch=False)
            else:
                wfo.status = 'trouble' if options.set_trouble else 'forget'
                wfi.notifyRequestor('Rejected by unified operator %s' %
                                    (comment),
                                    do_batch=False)
                session.commit()

        else:
            print "error in rejecting", wfo.name, results
Beispiel #30
0
def rejector(url, specific, options=None):

    up = componentInfo(soft=['wtc', 'jira'])
    #if not up.check(): return

    if specific and specific.startswith('/'):
        ## this is for a dataset
        print setDatasetStatus(specific, 'INVALID')
        return

    if options.filelist:
        wfs = []
        for line in filter(None, open(options.filelist).read().split('\n')):
            print line
            wfs.extend(
                session.query(Workflow).filter(
                    Workflow.name.contains(line)).all())
    elif specific:
        wfs = session.query(Workflow).filter(
            Workflow.name.contains(specific)).all()
        if not wfs:
            batches = batchInfo().content()
            for bname in batches:
                if specific == bname:
                    for pid in batches[bname]:
                        b_wfs = getWorkflowById(url, pid)
                        for wf in b_wfs:
                            wfs.append(
                                session.query(Workflow).filter(
                                    Workflow.name == wf).first())
                    break
    else:
        wfs = session.query(Workflow).filter(
            Workflow.status == 'assistance-clone').all()
        #wfs.extend( session.query(Workflow).filter(Workflow.status == 'assistance-reject').all())
        ## be careful then on clone case by case
        options.clone = True
        print "not supposed to function yet"
        return

    print len(wfs), "to reject"

    if len(wfs) > 1:
        print "\n".join([wfo.name for wfo in wfs])
        answer = raw_input('Reject these')
        if not answer.lower() in ['y', 'yes']:
            return

    for wfo in wfs:
        #wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject", spec
            return
        wfi = workflowInfo(url, wfo.name)

        comment = ""
        if options.comments: comment = ", reason: " + options.comments
        if options.keep:
            wfi.sendLog(
                'rejector',
                'invalidating the workflow by unified operator%s' % comment)
        else:
            wfi.sendLog(
                'rejector',
                'invalidating the workflow and outputs by unified operator%s' %
                comment)

        results = invalidate(url,
                             wfi,
                             only_resub=True,
                             with_output=(not options.keep))

        if all(results):
            print wfo.name, "rejected"
            if options and options.clone:
                wfo.status = 'trouble'
                session.commit()
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                schema['OriginalRequestName'] = wfo.name
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion'] = int(
                        schema['ProcessingVersion']
                    ) + 1  ## dubious str->int conversion
                else:
                    schema['ProcessingVersion'] = 2
                for k in schema.keys():
                    if k.startswith('Team'):
                        schema.pop(k)
                    if k.startswith('checkbox'):
                        schema.pop(k)

                ## a few tampering of the original request
                if options.Memory:
                    if schema['RequestType'] == 'TaskChain':
                        it = 1
                        while True:
                            t = 'Task%d' % it
                            it += 1
                            if t in schema:
                                schema[t]['Memory'] = options.Memory
                            else:
                                break
                    else:
                        schema['Memory'] = options.Memory

                if options.short_task and schema['RequestType'] == 'TaskChain':
                    translate = {}
                    it = 1
                    while True:
                        tt = 'Task%d' % it
                        if tt in schema:
                            tname = schema[tt]['TaskName']
                            ntname = 'T%d' % it
                            translate[tname] = ntname
                            it += 1
                            schema[tt]['TaskName'] = ntname
                            if 'InputTask' in schema[tt]:
                                itname = schema[tt]['InputTask']
                                schema[tt]['InputTask'] = translate[itname]
                        else:
                            break
                    for k in schema.get('ProcessingString', {}).keys():
                        schema['ProcessingString'][
                            translate[k]] = schema['ProcessingString'].pop(k)
                    for k in schema.get('AcquisitionEra', {}).keys():
                        schema['AcquisitionEra'][
                            translate[k]] = schema['AcquisitionEra'].pop(k)

                if options.Multicore:
                    ## to do : set it properly in taskchains
                    if schema['RequestType'] == 'TaskChain':
                        tasks, set_to = options.Multicore.split(
                            ':') if ':' in options.Multicore else (
                                "", options.Multicore)
                        set_to = int(set_to)
                        tasks = tasks.split(',') if tasks else ['Task1']
                        it = 1
                        while True:
                            tt = 'Task%d' % it
                            it += 1
                            if tt in schema:
                                tname = schema[tt]['TaskName']
                                if tname in tasks or tt in tasks:
                                    mem = schema[tt]['Memory']
                                    mcore = schema[tt].get('Multicore', 1)
                                    factor = (set_to / float(mcore))
                                    fraction_constant = 0.4
                                    mem_per_core_c = int(
                                        (1 - fraction_constant) * mem /
                                        float(mcore))
                                    print "mem per core", mem_per_core_c
                                    print "base mem", mem
                                    ## adjusting the parameter in the clone
                                    schema[tt]['Memory'] += (
                                        set_to - mcore) * mem_per_core_c
                                    schema[tt]['Multicore'] = set_to
                                    schema[tt]['TimePerEvent'] /= factor
                            else:
                                break
                    else:
                        schema['Multicore'] = options.Multicore
                if options.deterministic:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['DeterministicPileup'] = True
                if options.EventsPerJob:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['EventsPerJob'] = options.EventsPerJob
                    else:
                        schema['EventsPerJob'] = options.EventsPerJob
                if options.EventAwareLumiBased:
                    schema['SplittingAlgo'] = 'EventAwareLumiBased'
                if options.TimePerEvent:
                    schema['TimePerEvent'] = options.TimePerEvent

                if options.ProcessingString:
                    schema['ProcessingString'] = options.ProcessingString
                if options.AcquisitionEra:
                    schema['AcquisitionEra'] = options.AcquisitionEra
                if options.runs:
                    schema['RunWhitelist'] = map(int, options.runs.split(','))
                if options.PrepID:
                    schema['PrepID'] = options.PrepID

                if schema['RequestType'] == 'TaskChain' and options.no_output:
                    ntask = schema['TaskChain']
                    for it in range(1, ntask - 1):
                        schema['Task%d' % it]['KeepOutput'] = False
                    schema['TaskChain'] = ntask - 1
                    schema.pop('Task%d' % ntask)

                if options.priority:
                    schema['RequestPriority'] = options.priority

                ## update to the current priority
                schema['RequestPriority'] = wfi.request['RequestPriority']

                ## drop shit on the way to reqmgr2
                schema = reqMgrClient.purgeClonedSchema(schema)

                print "submitting"
                if (options.to_stepchain
                        and (schema['RequestType'] == 'TaskChain')):
                    ## transform the schema into StepChain schema
                    print "Transforming a TaskChain into a StepChain"
                    mcore = 0
                    mem = 0
                    schema['RequestType'] = 'StepChain'
                    schema['StepChain'] = schema.pop('TaskChain')
                    schema['SizePerEvent'] = 0
                    schema['TimePerEvent'] = 0
                    step = 1
                    s_n = {}
                    while True:
                        if 'Task%d' % step in schema:
                            sname = 'Step%d' % step
                            schema[sname] = schema.pop('Task%d' % step)
                            tmcore = schema[sname].pop('Multicore')
                            tmem = schema[sname].pop('Memory')
                            if mcore and tmcore != mcore:
                                wfi.sendLog(
                                    'rejector',
                                    'the conversion to stepchain encoutered different value of Multicore %d != %d'
                                    % (tmcore, mcore))
                                sendLog(
                                    'rejector',
                                    'the conversion of %s to stepchain encoutered different value of Multicore %d != %d'
                                    % (wfo.name, tmcore, mcore),
                                    level='critical')
                            mcore = max(mcore, tmcore)
                            mem = max(mem, tmem)
                            schema[sname]['StepName'] = schema[sname].pop(
                                'TaskName')
                            s_n[schema[sname]['StepName']] = sname
                            if 'InputTask' in schema[sname]:
                                schema[sname]['InputStep'] = schema[sname].pop(
                                    'InputTask')
                            eff = 1.
                            up_s = sname
                            while True:
                                ## climb up a step. supposedely already all converted
                                up_s = s_n.get(
                                    schema[up_s].get('InputStep', None), None)
                                if up_s:
                                    ## multiply with the efficiency
                                    eff *= schema[up_s].get(
                                        'FilterEfficiency', 1.)
                                else:
                                    ## or stop there
                                    break

                            if not 'KeepOutput' in schema[sname]:
                                ## this is a weird translation capability. Absence of keepoutput in step means : keep the output. while in TaskChain absence means : drop
                                schema[sname]['KeepOutput'] = False
                            schema['TimePerEvent'] += eff * schema[sname].pop(
                                'TimePerEvent')
                            schema['SizePerEvent'] += eff * schema[sname].pop(
                                'SizePerEvent')
                            step += 1
                        else:
                            break
                    schema['Multicore'] = mcore
                    schema['Memory'] = mem
                print json.dumps(schema, indent=2)
                newWorkflow = reqMgrClient.submitWorkflow(url, schema)
                if not newWorkflow:
                    msg = "Error in cloning {}".format(wfo.name)
                    print(msg)
                    wfi.sendLog('rejector', msg)

                    # Get the error message
                    time.sleep(5)
                    data = reqMgrClient.requestManagerPost(
                        url, "/reqmgr2/data/request", schema)
                    wfi.sendLog('rejector', data)

                    print json.dumps(schema, indent=2)
                    return
                print newWorkflow

                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfi.sendLog(
                    'rejector', 'Cloned into %s by unified operator %s' %
                    (newWorkflow, comment))
                #wfi.notifyRequestor('Cloned into %s by unified operator %s'%( newWorkflow, comment ),do_batch=False)
            else:
                wfo.status = 'trouble' if options.set_trouble else 'forget'
                wfi.notifyRequestor('Rejected by unified operator %s' %
                                    (comment),
                                    do_batch=False)
                session.commit()

        else:
            msg = "Error in rejecting {}: {}".format(wfo.name, results)
            print(msg)
            wfi.sendLog('rejector', msg)
Beispiel #31
0
def singleRecovery(url, task, initial, actions, do=False):
    payload = {
        "Requestor": os.getenv('USER'),
        "Group": 'DATAOPS',
        "RequestType": "Resubmission",
        "ACDCServer": initial['ConfigCacheUrl'],
        "ACDCDatabase": "acdcserver",
        "OriginalRequestName": initial['RequestName'],
        "OpenRunningTimeout": 0
    }
    copy_over = [
        'PrepID', 'Campaign', 'RequestPriority', 'TimePerEvent',
        'SizePerEvent', 'Group', 'Memory', 'RequestString', 'CMSSWVersion'
    ]
    for c in copy_over:
        if c in initial:
            payload[c] = copy.deepcopy(initial[c])
        else:
            print c, "not in the initial payload"

    #a massage ? boost the recovery over the initial wf
    payload['RequestPriority'] *= 2
    payload['RequestPriority'] = min(500000, payload['RequestPriority'])

    if actions:
        for action in actions:
            #if action.startswith('split'):
            #    factor = int(action.split('-')[-1]) if '-' in action else 2
            #    print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
            #    ## mention it's taking 2 times longer to have a 2 times finer splitting
            #    payload['TimePerEvent'] = factor*payload['TimePerEvent']
            if action.startswith('mem'):
                arg = action.split('-', 1)[-1]
                increase = set_to = None
                tasks, set_to = arg.split(':') if ':' in arg else (None, arg)
                tasks = tasks.split(',') if tasks else []
                if set_to.startswith('+'):
                    increase = int(set_to[1:])
                else:
                    set_to = int(set_to)
                ## increase the memory requirement by 1G

                if 'TaskChain' in initial:
                    mem_dict = {}
                    it = 1
                    while True:
                        t = 'Task%d' % it
                        it += 1
                        if t in initial:
                            tname = payload.setdefault(t,
                                                       initial[t])['TaskName']
                            mem = mem_dict.setdefault(tname,
                                                      payload[t]['Memory'])
                            if tasks and not tname in tasks:
                                print tname, "not concerned"
                                continue
                            if set_to:
                                mem_dict[tname] = set_to
                            else:
                                mem_dict[tname] += increase
                        else:
                            break
                    payload['Memory'] = mem_dict
                else:
                    payload['Memory'] = set_to
                #increase = int(action.split('-')[-1]) if '-' in action else 1000
                ## increase the memory requirement by 1G
                #payload['Memory'] += increase

            if action.startswith('split') and (
                    initial['RequestType'] in ['MonteCarlo'] or
                (initial['RequestType'] in ['TaskChain']
                 and not 'InputDataset' in initial['Task1'])):
                print "I should not be doing splitting for this type of request", initial[
                    'RequestName']
                return None
            if action.startswith('core'):
                arg = action.split('-', 1)[-1]
                tasks, set_to = arg.split(':') if ':' in arg else (None, arg)
                tasks = tasks.split(',') if tasks else []
                set_to = int(set_to)
                if 'TaskChain' in initial:
                    core_dict = {}
                    mem_dict = payload['Memory'] if type(
                        payload['Memory']) == dict else {}
                    it = 1
                    while True:
                        t = 'Task%d' % it
                        it += 1
                        if t in initial:
                            tname = payload.setdefault(t,
                                                       initial[t])['TaskName']
                            mcore = core_dict.setdefault(
                                tname, payload[t]['Multicore'])
                            mem = mem_dict.setdefault(tname,
                                                      payload[t]['Memory'])
                            if tasks and not tname in tasks:
                                print tname, "not concerned"
                                continue

                            factor = (set_to / float(mcore))
                            fraction_constant = 0.4
                            mem_per_core_c = int(
                                (1 - fraction_constant) * mem / float(mcore))
                            ##scale the memory
                            mem_dict[tname] += (set_to -
                                                mcore) * mem_per_core_c
                            ## scale time/event
                            time_dict[
                                tname] = payload[t]['TimePerEvent'] / factor
                            ## set the number of cores
                            core_dict[tname] = set_to
                        else:
                            break
                    payload['Multicore'] = core_dict
                    ##payload['TimePerEvent'] = time_dict ## cannot be used yet
                else:
                    payload['Multicore'] = increase

    acdc_round = 0
    initial_string = payload['RequestString']
    if initial_string.startswith('ACDC'):
        if initial_string[4].isdigit():
            acdc_round = int(initial_string[4])
        acdc_round += 1
        #print acdc_round
        #print "This is not allowed yet"
        #return None
    initial_string = initial_string.replace('ACDC_', '').replace(
        'ACDC%d_' % (acdc_round - 1), '')
    payload['RequestString'] = 'ACDC%d_%s' % (acdc_round, initial_string)
    payload['InitialTaskPath'] = task

    if not do:
        print json.dumps(payload, indent=2)
        return None

    print "ACDC payload"
    print json.dumps(payload, indent=2)
    print actions

    ## submit
    acdc = reqMgrClient.submitWorkflow(url, payload)
    if not acdc:
        print "Error in making ACDC for", initial["RequestName"]
        acdc = reqMgrClient.submitWorkflow(url, payload)
        if not acdc:
            print "Error twice in making ACDC for", initial["RequestName"]
            return None

    ## perform modifications
    if actions:
        for action in actions:
            if action.startswith('split'):
                factor = int(action.split('-')[-1]) if '-' in action else 2
                acdcInfo = workflowInfo(url, acdc)
                splittings = acdcInfo.getSplittings()
                for split in splittings:
                    for act in [
                            'avg_events_per_job', 'events_per_job',
                            'lumis_per_job'
                    ]:
                        if act in split:
                            print "Changing %s (%d) by a factor %d" % (
                                act, split[act], factor),
                            split[act] /= factor
                            print "to", split[act]
                            break
                    split['requestName'] = acdc
                    print "changing the splitting of", acdc
                    print json.dumps(split, indent=2)
                    print reqMgrClient.setWorkflowSplitting(url, acdc, split)

    data = reqMgrClient.setWorkflowApproved(url, acdc)
    print data
    return acdc
Beispiel #32
0
def singleClone(url, wfname, actions, comment, do=False):
    
    wfi = workflowInfo(url, wfname)
    payload = wfi.getSchema()
    initial = wfi.request

    payload['Requestor']           = os.getenv('USER')
    payload['Group']               = 'DATAOPS'
    payload['OriginalRequestName'] = initial['RequestName']
    payload['RequestPriority'] = initial['RequestPriority']

    if 'ProcessingVersion' in initial:
        payload['ProcessingVersion'] = int(initial['ProcessingVersion']) +1
    else:
        payload['ProcessingVersion'] = 2


## drop parameters on the way to reqmgr2
    paramBlacklist = ['BlockCloseMaxEvents', 'BlockCloseMaxFiles', 'BlockCloseMaxSize', 'BlockCloseMaxWaitTime',
                  'CouchWorkloadDBName', 'CustodialGroup', 'CustodialSubType', 'Dashboard',
                  'GracePeriod', 'HardTimeout', 'InitialPriority', 'inputMode', 'MaxMergeEvents', 'MaxMergeSize',
                  'MaxRSS', 'MaxVSize', 'MinMergeSize', 'NonCustodialGroup', 'NonCustodialSubType',
                  'OutputDatasets', 'ReqMgr2Only', 'RequestDate' 'RequestorDN', 'RequestName', 'RequestStatus',
                  'RequestTransition', 'RequestWorkflow', 'SiteWhitelist', 'SoftTimeout', 'SoftwareVersions',
                  'SubscriptionPriority', 'Team', 'timeStamp', 'TrustSitelists', 'TrustPUSitelists',
                  'TotalEstimatedJobs', 'TotalInputEvents', 'TotalInputLumis', 'TotalInputFiles','checkbox',
                  'DN', 'AutoApproveSubscriptionSites', 'NonCustodialSites', 'CustodialSites', 'OriginalRequestName', 'Teams', 'OutputModulesLFNBases', 
                  'SiteBlacklist', 'AllowOpportunistic', '_id']
    for p in paramBlacklist:
        if p in payload:
            payload.pop( p )
            pass

    if actions:
        for action in actions:
            if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
                if 'TaskChain' in payload:
                    print "Setting memory for clone of task chain"
                    it=1
                    while True:
                        t = 'Task%d'%it
                        it+=1
                        if t in payload:
                            payload[t]['Memory'] = actions[action]
                            print "Memory set for Task%d"%it
                        else:
                            break
                else:
                    print "Setting memory for non-taskchain workflow"
                    payload['Memory'] = actions[action]
                print "Memory set to " + actions[action]
                #This line is doesn't work for some reason
#                wfi.sendLog('actor','Memory of clone set to %d'%actions[action])

    print "Clone payload"
#    print json.dumps( payload , indent=2)
    print actions

    #Create clone
    clone = reqMgrClient.submitWorkflow(url, payload)
    if not clone:
        print "Error in making clone for",initial["RequestName"]
        clone = reqMgrClient.submitWorkflow(url, payload)
        if not clone:
            print "Error twice in making clone for",initial["RequestName"]
            sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"],level='critical')
            wfi.sendLog('actor','Failed to make a clone twice for %s!'%initial["RequestName"])
            return None

    if actions:
        for action in actions:
            if action.startswith('split'):
                cloneinfo = workflowInfo(url, clone)
                splittings = cloneinfo.getSplittings()
                if actions[action] != 'Same' and actions[action] != 'max' and actions[action] != '':
                    factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
                    for split in splittings:
                        for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                            if act in split:
                                wfi.sendLog('actor','Changing %s (%d) by a factor %d'%( act, split[act], factor))
                                print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
                                split[act] /= factor
                                print "to",split[act]
                                break
                        split['requestName'] = clone
                        print "changing the splitting of",clone
                        print json.dumps( split, indent=2 )
                        print reqMgrClient.setWorkflowSplitting(url, clone, split )
                elif 'max' in actions[action]:
                    for split in splittings:
                        for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                            if act in split:
                                wfi.sendLog('actor','Max splitting set for %s (%d'%( act, split[act]))
                                print "Changing %s (%d) "%( act, split[act]),
                                split[act] = 1
                                print "to max splitting ",split[act]
                                break
                        split['requestName'] = clone
                        print "changing the splitting of",clone
                        print json.dumps( split, indent=2 )
                        print reqMgrClient.setWorkflowSplitting(url, clone, split )

    #Approve
    data = reqMgrClient.setWorkflowApproved(url, clone)
    wfi.sendLog('actor','Cloned into %s'%clone)

#    wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
#    wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)

    print data
    return clone
def singleRecovery(url, task , initial, actions, do=False):
    payload = {
        "Requestor" : os.getenv('USER'),
        "Group" : 'DATAOPS',
        "RequestType" : "Resubmission",
        "ACDCServer" : "https://cmsweb.cern.ch/couchdb",
        "ACDCDatabase" : "acdcserver",
        "OriginalRequestName" : initial['RequestName']
        }
    copy_over = ['PrepID','RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ]        
    for c in copy_over:
        payload[c] = copy.deepcopy(initial[c])

    if actions:
        for action in actions:
            #if action.startswith('split'):
            #    factor = int(action.split('-')[-1]) if '-' in action else 2
            #    print "Changing time per event (%s) by a factor %d"%( payload['TimePerEvent'], factor)
            #    ## mention it's taking 2 times longer to have a 2 times finer splitting
            #    payload['TimePerEvent'] = factor*payload['TimePerEvent']
            if action.startswith('mem'):
                increase = int(action.split('-')[-1]) if '-' in action else 1000
                ## increase the memory requirement by 1G
                payload['Memory'] += increase

    if payload['RequestString'].startswith('ACDC'):
        print "This is not allowed yet"
        return None
    payload['RequestString'] = 'ACDC_'+payload['RequestString']
    payload['InitialTaskPath'] = task 

    if not do:
        print json.dumps( payload, indent=2)
        return None

    ## submit
    response = reqMgrClient.submitWorkflow(url, payload)
    m = re.search("details\/(.*)\'",response)
    if not m:
        print "Error in making ACDC for",initial["RequestName"]
        print response
        response = reqMgrClient.submitWorkflow(url, payload)
        m = re.search("details\/(.*)\'",response)
        if not m:
            print "Error twice in making ACDC for",initial["RequestName"]
            print response
            return None
    acdc = m.group(1)
    
    ## perform modifications
    if actions:
        for action in actions:
            if action.startswith('split'):
                factor = int(action.split('-')[-1]) if '-' in action else 2
                acdcInfo = workflowInfo(url, acdc)
                splittings = acdcInfo.getSplittings()
                for split in splittings:
                    for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                        if act in split:
                            print "Changing %s (%d) by a factor %d"%( act, split[act], factor),
                            split[act] /= factor
                            print "to",split[act]
                            break
                    split['requestName'] = acdc
                    print "changing the splitting of",acdc
                    print json.dumps( split, indent=2 )
                    print reqMgrClient.setWorkflowSplitting(url, split )
                
    data = reqMgrClient.setWorkflowApproved(url, acdc)
    print data
    return acdc
Beispiel #34
0
def rejector(url, specific, options=None):

    up = componentInfo()

    if specific and specific.startswith('/'):
        return

    if options.filelist:
        wfs = []
        for line in filter(None, open(options.filelist).read().split('\n')):
            print line
            wfs.extend( session.query(Workflow).filter(Workflow.name.contains(line)).all())
    elif specific:
        wfs = session.query(Workflow).filter(Workflow.name.contains(specific)).all()
    else:
        return 

    print len(wfs),"to reject"

    if len(wfs)>1:
        print "\n".join( [wfo.name for wfo in wfs] )
        answer = raw_input('Reject these')
        if not answer.lower() in ['y','yes']:
            return
        
    for wfo in wfs:
        #wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject",spec
            return
        results=[]
        wfi = workflowInfo(url, wfo.name)

        datasets = set(wfi.request['OutputDatasets'])
        reqMgrClient.invalidateWorkflow(url, wfo.name, current_status=wfi.request['RequestStatus'])

        wfi.sendLog('rejector','invalidating the workflow by unified operator')
        ## need to find the whole familly and reject the whole gang
        familly = getWorkflowById( url, wfi.request['PrepID'] , details=True)
        for fwl in familly:
            if fwl['RequestDate'] < wfi.request['RequestDate']: continue
            if fwl['RequestType']!='Resubmission': continue
            if 'OriginalRequestName' in fwl and fwl['OriginalRequestName'] != wfi.request['RequestName']: continue
            print "rejecting",fwl['RequestName']
            reqMgrClient.invalidateWorkflow(url, fwl['RequestName'], current_status=fwl['RequestStatus'])
            datasets.update( fwl['OutputDatasets'] )

        for dataset in datasets:
            if options.keep:
                print "keeping",dataset,"in its current status"
            else:
                results.append( setDatasetStatus(dataset, 'INVALID') )

        #if wfi.request['RequestStatus'] in ['rejected','rejected-archived','aborted','aborted-archived']:
        #    print 'already',wfi.request['RequestStatus']
        #    if not options.clone:
        #        wfo.status = 'forget'
        #        session.commit()
        #        continue

        if all(map(lambda result : result in ['None',None,True],results)):
            wfo.status = 'forget'
            session.commit()
            print wfo.name,"and",datasets,"are rejected"
            if options and options.clone:
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                schema['OriginalRequestName'] = wfo.name
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion'] = int(schema['ProcessingVersion'])+1 ## dubious str->int conversion
                else:
                    schema['ProcessingVersion']=2
                for k in schema.keys():
                    if k.startswith('Team'):
                        schema.pop(k)
                    if k.startswith('checkbox'):
                        schema.pop(k)

                ## a few tampering of the original request
                if options.Memory:
                    schema['Memory'] = options.Memory
                if options.deterministic:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['DeterministicPileup']  = True
                if options.EventsPerJob:
                    if schema['RequestType'] == 'TaskChain':
                        schema['Task1']['EventsPerJob'] = options.EventsPerJob
                    else:
                        schema['EventsPerJob'] = options.EventsPerJob
                if options.EventAwareLumiBased:
                    schema['SplittingAlgo'] = 'EventAwareLumiBased'
                if options.TimePerEvent:
                    schema['TimePerEvent'] = options.TimePerEvent

                if options.ProcessingString:
                    schema['ProcessingString'] = options.ProcessingString
                if options.AcquisitionEra:
                    schema['AcquisitionEra'] = options.AcquisitionEra
                if options.runs:
                    schema['RunWhitelist'] = map(int,options.runs.split(','))
                if options.PrepID:
                    schema['PrepID'] =options.PrepID

                if schema['RequestType'] == 'TaskChain' and options.no_output:
                    ntask = schema['TaskChain']
                    for it in range(1,ntask-1):
                        schema['Task%d'%it]['KeepOutput'] = False
                    schema['TaskChain'] = ntask-1
                    schema.pop('Task%d'%ntask)

                ## update to the current priority
                schema['RequestPriority'] = wfi.request['RequestPriority']

                ## drop shit on the way to reqmgr2
                for p in ['RequestStatus',
                          'RequestTransition',
                          'RequestorDN',
                          'RequestWorkflow',
                          'OutputDatasets',
                          'ReqMgr2Only',
                          #'Group',
                          'RequestDate',
                          #'ConfigCacheUrl',
                          'RequestName',
                          'timeStamp',
                          'SoftwareVersions',
                          'CouchURL'
                          ]:
                    if p in schema:
                        schema.pop( p )
                        #pass
                print "submitting"
                print json.dumps( schema, indent=2 )
                newWorkflow = reqMgrClient.submitWorkflow(url, schema)
                if not newWorkflow:
                    print "error in cloning",wfo.name
                    print json.dumps( schema, indent=2 )
                    return 
                print newWorkflow
                #m = re.search("details\/(.*)\'",response)
                #if not m:
                #    print "error in cloning",wfo.name
                #    print response
                #    print json.dumps( schema, indent=2 )
                #    return 
                #newWorkflow = m.group(1)

                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfo.status = 'trouble'
                session.commit()
                wfi.sendLog('rejector','Cloned into %s by unified operator'%( newWorkflow ))
        else:
            print "error in rejecting",wfo.name,results
Beispiel #35
0
def singleClone(url, wfname, actions, comment, do=False):

    wfi = workflowInfo(url, wfname)
    payload = wfi.getSchema()
    initial = wfi.request

    payload['Requestor'] = os.getenv('USER')
    payload['Group'] = 'DATAOPS'
    payload['OriginalRequestName'] = initial['RequestName']
    payload['RequestPriority'] = initial['RequestPriority']

    if 'ProcessingVersion' in initial:
        payload['ProcessingVersion'] = int(initial['ProcessingVersion']) + 1
    else:
        payload['ProcessingVersion'] = 2

    payload = reqMgrClient.purgeClonedSchema(payload)

    if actions:
        for action in actions:
            if action.startswith('mem') and actions[action] != "" and actions[
                    action] != 'Same':
                if 'TaskChain' in payload:
                    print "Setting memory for clone of task chain"
                    mem_dict = {}
                    it = 1
                    while True:
                        t = 'Task%d' % it
                        it += 1
                        if t in payload:
                            tname = payload[t]['TaskName']
                            mem_dict[tname] = int(actions[action])
                            print "Memory set for Task%d" % it
                        else:
                            break
                    payload['Memory'] = mem_dict
                else:
                    print "Setting memory for non-taskchain workflow"
                    payload['Memory'] = int(actions[action])
                print "Memory set to " + actions[action]

    print "Clone payload"
    #    print json.dumps( payload , indent=2)
    print actions

    #Create clone
    clone = reqMgrClient.submitWorkflow(url, payload)
    if not clone:
        print "Error in making clone for", initial["RequestName"]
        clone = reqMgrClient.submitWorkflow(url, payload)
        if not clone:
            print "Error twice in making clone for", initial["RequestName"]
            sendLog('actor',
                    'Failed to make a clone twice for %s!' %
                    initial["RequestName"],
                    level='critical')
            wfi.sendLog(
                'actor', 'Failed to make a clone twice for %s!' %
                initial["RequestName"])
            return None

    if actions:
        for action in actions:
            if action.startswith('split'):
                cloneinfo = workflowInfo(url, clone)
                splittings = cloneinfo.getSplittingsNew(strip=True)
                if actions[action] != 'Same' and actions[
                        action] != 'max' and actions[action] != '':
                    factor = int(
                        actions[action][0:-1]) if 'x' in actions[action] else 2
                    for split in splittings:
                        split_par = split['splitParams']
                        for act in [
                                'avg_events_per_job', 'events_per_job',
                                'lumis_per_job'
                        ]:
                            if act in split_par:
                                wfi.sendLog(
                                    'actor',
                                    'Changing %s (%d) by a factor %d' %
                                    (act, split_par[act], factor))
                                split_par[act] /= factor
                                print "to", split_par[act]
                                break
                        #split['requestName'] = clone
                        #print "changing the splitting of",clone
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, clone, split )
                elif 'max' in actions[action]:
                    for split in splittings:
                        split_par = split['splitParams']
                        for act in [
                                'avg_events_per_job', 'events_per_job',
                                'lumis_per_job'
                        ]:
                            if act in split_par:
                                wfi.sendLog(
                                    'actor', 'Max splitting set for %s (%d' %
                                    (act, split_par[act]))
                                print "Changing %s (%d) " % (act,
                                                             split_par[act]),
                                split_par[act] = 1
                                print "to max splitting ", split_par[act]
                                break
                        #split['requestName'] = clone
                        #print "changing the splitting of",clone
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, clone, split )

                print "changing the splitting of", clone
                print json.dumps(splittings, indent=2)
                print reqMgrClient.setWorkflowSplitting(url, clone, splittings)
    #Approve
    data = reqMgrClient.setWorkflowApproved(url, clone)
    #wfi.sendLog('actor','Cloned into %s'%clone)

    #    wfi.sendLog('actor','Cloned into %s by unified operator %s'%( clone, comment ))
    #    wfi.notifyRequestor('Cloned into %s by unified operator %s'%( clone, comment ),do_batch=False)

    print data
    return clone
Beispiel #36
0
def singleRecovery(url, task, initial, actions, do=False):
    print "Inside single recovery!"
    payload = {
        "Requestor" : os.getenv('USER'),
        "Group" : 'DATAOPS',
        "RequestType" : "Resubmission",
        "ACDCServer" : initial['ConfigCacheUrl'],
        "ACDCDatabase" : "acdcserver",
        "OriginalRequestName" : initial['RequestName'],
        "OpenRunningTimeout" : 0
    }
    copy_over = ['PrepID','Campaign','RequestPriority', 'TimePerEvent', 'SizePerEvent', 'Group', 'Memory', 'RequestString' ,'CMSSWVersion']
    for c in copy_over:
        if c in initial:
            payload[c] = copy.deepcopy(initial[c])
        else:
            print c,"not in the initial payload"

    #a massage ? boost the recovery over the initial wf
#    payload['RequestPriority'] *= 10
    #Max priority is 1M
    payload['RequestPriority'] = min(500000,  payload['RequestPriority']*2 ) ## never above 500k

    #change parameters based on actions here
    if actions:
        for action in actions:
            if action.startswith('mem') and actions[action] != "" and actions[action] != 'Same':
                #if multicore parameter is also used, need to scale memory by the new number of cores
                if 'multicore' in actions and actions['multicore'] != "":
                    continue
                ## Taskchains needs to be treated special to set the memory to all tasks
                set_to = int(actions[action])
                if 'TaskChain' in initial:
                    mem_dict = {}
                    it = 1
                    while True:
                        t = 'Task%d'%it
                        it += 1
                        if t in initial:
                            tname = payload.setdefault(t, initial[t])['TaskName']
                            mem = mem_dict.setdefault( tname, payload[t]['Memory'])
                            mem_dict[tname] = set_to
                        else:
                            break
                    payload['Memory'] = mem_dict
                    print "Memory set to: ",json.dumps( mem_dict, indent=2)
                else: 
                    payload['Memory'] = set_to
                    print "Memory set to: ", set_to


            if action.startswith('multicore') and actions[action] != "":
                set_to = int(actions[action] )
                ## Taskchains needs to be treated special to set the multicore and memory values to all tasks
                if 'TaskChain' in initial:
                    mem_dict  = payload['Memory'] if type(payload['Memory'])==dict else {}
                    core_dict = {}
                    it = 1
                    while True:
                        t = 'Task%d'%it
                        it += 1
                        if t in initial:
                            tname = payload.setdefault(t, initial[t])['TaskName']
                            mem = mem_dict.setdefault( tname, payload[t]['Memory'])

                            #Need to scale the memory by the new number of cores
                            initial_cores = payload[t].setdefault('Multicore', 1) 

                            if 'memory' in actions and actions['memory'] != "" and actions['memory'] != 'Same':
                                mem = actions['memory']

                            fraction_constant = 0.4
                            mem_per_core_c = int (( 1 - fraction_constant) * mem / float(initial_cores) )
                            
                            mem_dict[tname] = int ( mem + (set_to - initial_cores)*mem_per_core_c )
                            core_dict[tname] = set_to

                            print "For ", t
                            print "Multicore set to ", set_to
                            print "Memory set to ", mem_dict[tname]
                        else:
                           break
                    payload['Memory']    = mem_dict
                    payload['Multicore'] = core_dict
                                        
                else:
                    #Need to scale the memory by the new number of cores
                    initial_cores = initial.setdefault('Multicore', 1) 

                    mem = payload['Memory']
                    if 'memory' in actions and actions['memory'] != "" and actions['memory'] != 'Same' :
                        mem = actions['memory']

                    fraction_constant = 0.4
                    mem_per_core_c = int (( 1 - fraction_constant) * mem / float(initial_cores) )

                    payload['Multicore'] = set_to
                    payload['Memory'] = int ( mem + (set_to - initial_cores)*mem_per_core_c )

                    print "Multicore set to ", set_to
                    print "Memory set to ", payload['Memory']


            if action.startswith('split'):
                
                split_alert = (initial['RequestType'] in ['MonteCarlo'] )
                for key in initial:
                    if key == 'SplittingAlgo' and (initial[key] in ['EventBased']):
                        split_alert = True
                    elif key.startswith('Task') and key != 'TaskChain':
                        for key2 in initial[key]:
                            if key2 == 'TaskName':
                                this_taskname = initial[key][key2]
                                recover_task = task.split('/')[-1]
                                print "For recovery of task",recover_task
                                print "Looking at task",this_taskname
                                if (recover_task == this_taskname) and (initial[key]['SplittingAlgo'] in ['EventBased']):
                                    ## the task to be recovered is actually of the wrong type to allow change of splitting
                                    sendLog('actor','To recover on %s, changing the splitting on %s is not really allowed and this will be ignored instead of failing acdc.'%( task, initial[key]['SplittingAlgo']), level='critical')
                                    ## do not send an alert and stop the acdc
                                    #split_alert = True

                if split_alert:
                    sendLog('actor','Cannot change splitting for %s'%initial['RequestName'],level='critical')
                    print "I should not be doing splitting for this type of request",initial['RequestName']
                    return None

    acdc_round = 0
    initial_string = payload['RequestString']
    if initial_string.startswith('ACDC'):
        if initial_string[4].isdigit():
            acdc_round = int(initial_string[4])
        acdc_round += 1

    initial_string = initial_string.replace('ACDC_','').replace('ACDC%d_'%(acdc_round-1),'')
    payload['RequestString'] = 'ACDC%d_%s'%(acdc_round,initial_string)
    payload['InitialTaskPath'] = task

    if not do:
        print json.dumps( payload, indent=2)
        return None

    print "ACDC payload"
#    print json.dumps( payload , indent=2)
    print actions

    ## submit here
    acdc = reqMgrClient.submitWorkflow(url, payload)
    if not acdc:
        print "Error in making ACDC for",initial["RequestName"]
        acdc = reqMgrClient.submitWorkflow(url, payload)
        if not acdc:
            print "Error twice in making ACDC for",initial["RequestName"]
            sendLog('actor','Failed twice in making ACDCs for %s!'%initial['RequestName'],level='critical')                
            return None


    ## change splitting if requested
    if actions:
        for action in actions:
            if action.startswith('split'):
                acdcInfo = workflowInfo(url, acdc)
                splittings = acdcInfo.getSplittingsNew(strip=True)
                if actions[action] != 'Same' and actions[action] != 'max':
                    factor = int(actions[action][0:-1]) if 'x' in actions[action] else 2
                    for split in splittings:
                        split_par = split['splitParams']
                        if split['splitAlgo'] in ['EventBased']:
                            sendLog('actor',"Changing the splitting on %s for %s is not permitted. Not changing."%(split['splitAlgo'],initial["RequestName"]), level='critical')
                            continue
                        for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                            if act in split_par:
                                print "Changing %s (%d) by a factor %d"%( act, split_par[act], factor),
                                split_par[act] /= factor
                                print "to",split_par[act]
                                break
                        #split['requestName'] = acdc
                        #print "changing the splitting of",acdc
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, acdc, split )

                elif 'max' in actions[action]:
                    for split in splittings:
                        split_par = split['splitParams']
                        for act in ['avg_events_per_job','events_per_job','lumis_per_job']:
                            if act in split_par:
                                print "Changing %s (%d) "%( act, split_par[act]),
                                split_par[act] = 1
                                print "to max splitting ",split_par[act]
                                break
                        #split['requestName'] = acdc
                        #print "changing the splitting of",acdc
                        #print json.dumps( split, indent=2 )
                        #print reqMgrClient.setWorkflowSplitting(url, acdc, split )
                print "changing the splitting of",acdc
                print json.dumps( splittings, indent=2 )                
                done = reqMgrClient.setWorkflowSplitting(url, acdc, splittings )
                ## check on done == True
                

    data = reqMgrClient.setWorkflowApproved(url, acdc)
    

    print data
    return acdc