Exemplo n.º 1
0
def main():
    url='cmsweb.cern.ch'
    
    #Create option parser
    usage = "\n       python %prog [-f FILE_NAME | WORKFLOW_NAME ...]\n"
    parser = OptionParser(usage=usage)
    parser.add_option('-f', '--file', help='Text file with a list of workflows', dest='file')
    parser.add_option('-i', '--invalidate', action='store_true', default=False,
                      help='Also invalidate output datasets on DBS', dest='invalidate')
    (options, args) = parser.parse_args()
    
    if options.file:
        wfs = [l.strip() for l in open(options.file) if l.strip()]
    elif args:
        wfs = args
    else:
        parser.error("Provide the workflow of a file of workflows")
        sys.exit(1)
    
    for wf in wfs:
        print "Aborting workflow: " + wf
        reqMgrClient.abortWorkflow(url, wf)
        print "Aborted"

        if options.invalidate:
            print "Invalidating datasets"
            datasets = reqMgrClient.outputdatasetsWorkflow(url, wf)
            for ds in datasets:
                print ds
                dbs3.setDatasetStatus(ds, 'INVALID', files=True)

    sys.exit(0);
def main():
    """
    Read the text file, for each workflow try:
    First abort it, then clone it.
    """
    args=sys.argv[1:]
    if not len(args)==3:
        print "usage:abortAndClone file.txt user group"
        sys.exit(0)
    filename = args[0]
    user = args[1]
    group = args[2]

    #reading workflow list
    workflows = [wf.strip() for wf in open(filename).readlines() if wf.strip()]
    for workflow in workflows:
        #abort workflow
        print "Aborting workflow: " + workflow
        reqMgrClient.abortWorkflow(url, workflow)
        print "Aborted. Now cloning workflow..."
            
        #invalidates datasets
        print "Invalidating datasets"
        
        datasets = reqMgrClient.outputdatasetsWorkflow(url, workflow)
        for dataset in datasets:
            print dataset
            dbs3.setStatusDBS3(dbs3_url, dataset, 'INVALID', None)

        #clone workflow
        clone = resubmit.cloneWorkflow(workflow, user, group)
        print "Cloned workflow: ",   clone
    sys.exit(0);
Exemplo n.º 3
0
def rejector(url, specific ):

    if specific.startswith('/'):
        pass
    else:
        wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject",spec
            return
        results=[]
        wl = getWorkLoad(url, wfo.name)
        if wl['RequestStatus'] in ['assignment-approved','new']:
            #results.append( reqMgrClient.rejectWorkflow(url, wfo.name))
            reqMgrClient.rejectWorkflow(url, wfo.name)
        else:
            #results.append( reqMgrClient.abortWorkflow(url, wfo.name))
            reqMgrClient.abortWorkflow(url, wfo.name)
        
        datasets = wl['OutputDatasets']
        for dataset in datasets:
            results.append( setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', dataset, 'INVALID', None) )
        if all(map(lambda result : result in ['None',None],results)):
            wfo.status = 'forget'
            session.commit()
            print wfo.name,"and",datasets,"are rejected"
        else:
            print "error in rejecting",wfo.name,results
Exemplo n.º 4
0
def main():
    """
    Read the text file, for each workflow try:
    First abort it, then clone it.
    """
    usage = "\n       python %prog [options] [WORKFLOW_NAME] [USER GROUP]\n"\
            "WORKFLOW_NAME: if the list file is provided this should be empty\n"\
            "USER: the user for creating the clone, if empty it will\n"\
            "      use the OS user running the script\n"\
            "GROUP: the group for creating the clone, if empty it will\n"\
            "      use 'DATAOPS' by default"

    parser = OptionParser(usage=usage)
    parser.add_option('-f', '--file', help='Text file of workflows to Abort and Clone', dest='file')
    (options, args) = parser.parse_args()
    
    # Check the arguments, get info from them
    if options.file:
        wfs = [l.strip() for l in open(options.file) if l.strip()]
        if len(args) == 2:
            user = args[0]
            group = args[1]
        elif len(args) == 0:
            #get os username by default
            uinfo = pwd.getpwuid(os.getuid())
            user = uinfo.pw_name
            #group by default DATAOPS
            group = 'DATAOPS'
    else:
        if len(args) == 3:
            user = args[1]
            group = args[2]
        elif len(args) == 1:
            #get os username by default
            uinfo = pwd.getpwuid(os.getuid())
            user = uinfo.pw_name
            #group by default DATAOPS
            group = 'DATAOPS'
        else:
            parser.error("Provide the workflow of a file of workflows")
            sys.exit(1)
        #name of workflow
        wfs = [args[0]]

    for wf in wfs:
        #abort workflow
        print "Aborting workflow: " + wf
        reqMgrClient.abortWorkflow(url, wf)
        #invalidates datasets
        print "Invalidating datasets"
        datasets = reqMgrClient.outputdatasetsWorkflow(url, wf)
        for ds in datasets:
            print ds
            dbs3.setDatasetStatus(ds, 'INVALID', files=True)

        #clone workflow
        clone = resubmit.cloneWorkflow(wf, user, group)
    sys.exit(0);
Exemplo n.º 5
0
def rejector(url, specific, options=None):

    if specific.startswith('/'):
        pass
    else:
        wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject", spec
            return
        results = []
        wfi = workflowInfo(url, wfo.name)
        if wfi.request['RequestStatus'] in [
                'assignment-approved', 'new', 'completed'
        ]:
            #results.append( reqMgrClient.rejectWorkflow(url, wfo.name))
            reqMgrClient.rejectWorkflow(url, wfo.name)
        else:
            #results.append( reqMgrClient.abortWorkflow(url, wfo.name))
            reqMgrClient.abortWorkflow(url, wfo.name)

        datasets = wfi.request['OutputDatasets']
        for dataset in datasets:
            if options.keep:
                print "keeping", dataset, "in its current status"
            else:
                results.append(
                    setDatasetStatusDBS3.setStatusDBS3(
                        'https://cmsweb.cern.ch/dbs/prod/global/DBSWriter',
                        dataset, 'INVALID', None))

        if all(map(lambda result: result in ['None', None], results)):
            wfo.status = 'forget'
            session.commit()
            print wfo.name, "and", datasets, "are rejected"
            if options and options.clone:
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion'] += 1
                else:
                    schema['ProcessingVersion'] = 2
                ##schema.pop('RequestDate') ## ok then, let's not reset the time stamp
                if options.Memory:
                    schema['Memory'] = options.Memory
                response = reqMgrClient.submitWorkflow(url, schema)
                m = re.search("details\/(.*)\'", response)
                if not m:
                    print "error in cloning", wfo.name
                    print response
                    return
                newWorkflow = m.group(1)
                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfo.status = 'trouble'
                session.commit()
        else:
            print "error in rejecting", wfo.name, results
Exemplo n.º 6
0
def rejector(url, specific, options=None):
    
    if specific.startswith('/'):
        pass
    else:
        wfo = session.query(Workflow).filter(Workflow.name == specific).first()
        if not wfo:
            print "cannot reject",spec
            return
        results=[]
        wfi = workflowInfo(url, wfo.name)
        if wfi.request['RequestStatus'] in ['assignment-approved','new','completed']:
            #results.append( reqMgrClient.rejectWorkflow(url, wfo.name))
            reqMgrClient.rejectWorkflow(url, wfo.name)
        else:
            #results.append( reqMgrClient.abortWorkflow(url, wfo.name))
            reqMgrClient.abortWorkflow(url, wfo.name)
        
        datasets = wfi.request['OutputDatasets']
        for dataset in datasets:
            if options.keep:
                print "keeping",dataset,"in its current status"
            else:
                results.append( setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', dataset, 'INVALID', None) )

        if all(map(lambda result : result in ['None',None],results)):
            wfo.status = 'forget'
            session.commit()
            print wfo.name,"and",datasets,"are rejected"
            if options and options.clone:
                schema = wfi.getSchema()
                schema['Requestor'] = os.getenv('USER')
                schema['Group'] = 'DATAOPS'
                if 'ProcessingVersion' in schema:
                    schema['ProcessingVersion']+=1
                else:
                    schema['ProcessingVersion']=2
                ##schema.pop('RequestDate') ## ok then, let's not reset the time stamp
                if options.Memory:
                    schema['Memory'] = options.Memory
                response = reqMgrClient.submitWorkflow(url, schema)
                m = re.search("details\/(.*)\'",response)
                if not m:
                    print "error in cloning",wfo.name
                    print response
                    return 
                newWorkflow = m.group(1)
                data = reqMgrClient.setWorkflowApproved(url, newWorkflow)
                print data
                wfo.status = 'trouble'
                session.commit()
        else:
            print "error in rejecting",wfo.name,results
def main():
    """
    Read the text file, for each workflow try:
    Abort it.
    """
    args=sys.argv[1:]
    if not len(args)==1:
        print "usage:abortWorkflows file.txt"
        sys.exit(0)
    url='cmsweb.cern.ch'
    filename=args[0]
    workflows = [wf.strip() for wf in open(filename).readlines() if wf.strip()]
    for workflow in workflows:
        print "Aborting workflow: " + workflow
        reqMgrClient.abortWorkflow(url, workflow)
        print "Aborted"
    sys.exit(0);
Exemplo n.º 8
0
def main():
    """
    Read the text file, for each workflow try:
    Abort it.
    """
    args = sys.argv[1:]
    if not len(args) == 1:
        print "usage:abortWorkflows file.txt"
        sys.exit(0)
    url = 'cmsweb.cern.ch'
    filename = args[0]
    workflows = [wf.strip() for wf in open(filename).readlines() if wf.strip()]
    for workflow in workflows:
        print "Aborting workflow: " + workflow
        reqMgrClient.abortWorkflow(url, workflow)
        print "Aborted"
    sys.exit(0)
Exemplo n.º 9
0
def main():
    url = 'cmsweb.cern.ch'

    #Create option parser
    usage = "\n       python %prog [-f FILE_NAME | WORKFLOW_NAME ...]\n"
    parser = OptionParser(usage=usage)
    parser.add_option('-f',
                      '--file',
                      help='Text file with a list of workflows',
                      dest='file')
    parser.add_option('-i',
                      '--invalidate',
                      action='store_true',
                      default=False,
                      help='Also invalidate output datasets on DBS',
                      dest='invalidate')
    (options, args) = parser.parse_args()

    if options.file:
        wfs = [l.strip() for l in open(options.file) if l.strip()]
    elif args:
        wfs = args
    else:
        parser.error("Provide the workflow of a file of workflows")
        sys.exit(1)

    for wf in wfs:
        print "Aborting workflow: " + wf
        reqMgrClient.abortWorkflow(url, wf)
        print "Aborted"

        if options.invalidate:
            print "Invalidating datasets"
            datasets = reqMgrClient.outputdatasetsWorkflow(url, wf)
            for ds in datasets:
                print ds
                dbs3.setDatasetStatus(ds, 'INVALID', files=True)

    sys.exit(0)
Exemplo n.º 10
0
def invalidator(url, invalid_status='INVALID'):
    mcm = McMClient(dev=False)
    invalids = mcm.getA('invalidations',query='status=new')
    print len(invalids),"Object to be invalidated"
    for invalid in invalids:
        acknowledge= False
        if invalid['type'] == 'request':
            wfn = invalid['object']
            print "need to invalidate the workflow",wfn
            wfo = session.query(Workflow).filter(Workflow.name == wfn).first()
            if wfo:
                ## set forget of that thing (although checkor will recover from it)
                wfo.status = 'forget'
                session.commit()
            wfi = workflowInfo(url, wfn)
            success = "not rejected"
            if wfi.request['RequestStatus'] in ['assignment-approved','new','completed']:
                success = reqMgrClient.rejectWorkflow(url, wfn)
                pass
            else:
                success = reqMgrClient.abortWorkflow(url, wfn)
                pass
            print success
            acknowledge= True
        elif invalid['type'] == 'dataset':
            dataset = invalid['object']
            if 'None-' in dataset: continue
            if 'FAKE-' in dataset: continue
            print "setting",dataset,"to",invalid_status
            success = "not invalidated"
            success = setDatasetStatusDBS3.setStatusDBS3('https://cmsweb.cern.ch/dbs/prod/global/DBSWriter', dataset, invalid_status, None)
            print success
            ## make a delete request from everywhere we can find ?
            acknowledge= True
        else:
            print "\t\t",invalid['type']," type not recognized"

        if acknowledge:
            ## acknoldge invalidation in mcm, provided we can have the api
            print "No acknowledgment api yet available"
Exemplo n.º 11
0
 def testAbortWorkflow(self):
     # abortWorkflow
     te = reqMgr.abortWorkflow(url, testwf)
     if te is None:
         self.fail("None output")
Exemplo n.º 12
0
def main():
    """
    Read the text file, for each workflow try:
    First abort it, then clone it.
    """
    usage = "\n       python %prog [options] [WORKFLOW_NAME] [USER GROUP]\n"\
            "WORKFLOW_NAME: if the list file is provided this should be empty\n"\
            "USER: the user for creating the clone, if empty it will\n"\
            "      use the OS user running the script\n"\
            "GROUP: the group for creating the clone, if empty it will\n"\
            "      use 'DATAOPS' by default"

    parser = OptionParser(usage=usage)
    parser.add_option('-f',
                      '--file',
                      help='Text file of workflows to Abort and Clone',
                      dest='file')
    (options, args) = parser.parse_args()

    # Check the arguments, get info from them
    if options.file:
        wfs = [l.strip() for l in open(options.file) if l.strip()]
        if len(args) == 2:
            user = args[0]
            group = args[1]
        elif len(args) == 0:
            #get os username by default
            uinfo = pwd.getpwuid(os.getuid())
            user = uinfo.pw_name
            #group by default DATAOPS
            group = 'DATAOPS'
    else:
        if len(args) == 3:
            user = args[1]
            group = args[2]
        elif len(args) == 1:
            #get os username by default
            uinfo = pwd.getpwuid(os.getuid())
            user = uinfo.pw_name
            #group by default DATAOPS
            group = 'DATAOPS'
        else:
            parser.error("Provide the workflow of a file of workflows")
            sys.exit(1)
        #name of workflow
        wfs = [args[0]]

    for wf in wfs:
        #abort workflow
        print "Aborting workflow: " + wf
        reqMgrClient.abortWorkflow(url, wf)
        #invalidates datasets
        print "Invalidating datasets"
        datasets = reqMgrClient.outputdatasetsWorkflow(url, wf)
        for ds in datasets:
            print ds
            dbs3.setDatasetStatus(ds, 'INVALID', files=True)

        #clone workflow
        clone = resubmit.cloneWorkflow(wf, user, group)
    sys.exit(0)
Exemplo n.º 13
0
 def testAbortWorkflow(self):
     # abortWorkflow
     te = reqMgr.abortWorkflow(url, testwf)
     if te is None:
         self.fail("None output")
Exemplo n.º 14
0
        if len(j1) != 1:
            os.system('echo '+wf[0]+' | mail -s \"batch_killor.py error 3\" [email protected]')
            sys.exit(1)

        j1 = j1[0]

        j1 = j1[workflow]

        status= j1['RequestStatus']

        if status == "completed" or status == "assignment-approved" or status == "assigned" or status == "running-open" or status == "running-closed" or status == "acquired" or status == "failed":        
            if status == "completed" or status == "assignment-approved" or status == "failed":
                reqMgrClient.rejectWorkflow(url,workflow)
            if status == "assigned" or status == "running-open" or status == "running-closed" or status == "acquired":
                reqMgrClient.abortWorkflow(url,workflow)
        elif status != "aborted-archived" and status != "aborted" and status != "rejected" and status != "rejected-archived" and status != "normal-archived":

            os.system('echo '+workflow+' | mail -s \"batch_killor.py error 1\" [email protected] --')
            sys.exit(1)
        else:
            continue

        #headers={"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"}
        #params = {"requestName" : workflow,"status" : newstatus}
        #encodedParams = urllib.urlencode(params)
        #conn.request("PUT", "/reqmgr/reqMgr/request", encodedParams, headers)
        #response = conn.getresponse()
        #print response.status, response.reason
    #    #curs.execute("insert into workflows_archive VALUES "+str(workflow_row)+";")
Exemplo n.º 15
0
                'echo ' + wf[0] +
                ' | mail -s \"batch_killor.py error 3\" [email protected]'
            )
            sys.exit(1)

        j1 = j1[0]

        j1 = j1[workflow]

        status = j1['RequestStatus']

        if status == "completed" or status == "assignment-approved" or status == "assigned" or status == "running-open" or status == "running-closed" or status == "acquired":
            if status == "completed" or status == "assignment-approved":
                reqMgrClient.rejectWorkflow(url, workflow)
            if status == "assigned" or status == "running-open" or status == "running-closed" or status == "acquired":
                reqMgrClient.abortWorkflow(url, workflow)
        elif status != "aborted-archived" and status != "aborted" and status != "rejected" and status != "rejected-archived":
            os.system(
                'echo ' + workflow +
                ' | mail -s \"batch_killor.py error 1\" [email protected] --'
            )
            sys.exit(1)
        else:
            continue

        #headers={"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"}
        #params = {"requestName" : workflow,"status" : newstatus}
        #encodedParams = urllib.urlencode(params)
        #conn.request("PUT", "/reqmgr/reqMgr/request", encodedParams, headers)
        #response = conn.getresponse()
        #print response.status, response.reason