Beispiel #1
0
    def testResubmit(self):
        s = resubmit(self.logger, [])

        #1) missing required -d option
        expRes = CommandResult(2001, 'ERROR: Task option is required')
        res = s()
        self.assertEquals(expRes, res)

        #2) correct execution
        analysisDir = self.reqarea
        s = resubmit(self.logger, self.maplistopt + ["-d", analysisDir])
        res = s()
        expRes = CommandResult(0, '')
        self.assertEquals( expRes, res)

        #3) wrong -d option
        analysisDir = os.path.join(os.path.dirname(__file__), 'crab_XXX')
        self.assertRaises( TaskNotFoundException, resubmit, self.logger, self.maplistopt + ["-d", analysisDir])
Beispiel #2
0
    def testResubmit(self):
        s = resubmit(self.logger, [])

        #1) missing required -d option
        expRes = CommandResult(2001, 'ERROR: Task option is required')
        res = s()
        self.assertEquals(expRes, res)

        #2) correct execution
        analysisDir = self.reqarea
        s = resubmit(self.logger, self.maplistopt + ["-d", analysisDir])
        res = s()
        expRes = CommandResult(0, '')
        self.assertEquals(expRes, res)

        #3) wrong -d option
        analysisDir = os.path.join(os.path.dirname(__file__), 'crab_XXX')
        self.assertRaises(TaskNotFoundException, resubmit, self.logger,
                          self.maplistopt + ["-d", analysisDir])
def resubmit_crab(args):
    '''Resubmit jobs'''
    if not crabLoaded:
        logging.error('You must source a crab environment to submit to crab.\nsource /cvmfs/cms.cern.ch/crab3/crab.sh')
        return
    crab_dirs = []
    if args.jobName:
        workArea = get_crab_workArea(args)
        crab_dirs += sorted(glob.glob('{0}/*'.format(workArea)))
    elif args.directories:
        for d in args.directories:
            crab_dirs += glob.glob(d)
    else:
        log.error("Shouldn't be possible to get here")

    tblogger, logger, memhandler = initLoggers()
    tblogger.setLevel(logging.WARNING)
    logger.setLevel(logging.WARNING)
    memhandler.setLevel(logging.WARNING)

    resubmitMap = {}
    for d in crab_dirs:
        if os.path.exists(d):
            statusArgs = ['--dir',d]
            resubmitArgs = ['--dir',d]
            try:
                summary = crabClientStatus.status(logger,statusArgs)()
                resubmit = False
                total = 0
                failed = 0
                allJobStatus = {}
                if 'jobs' in summary:
                    for j,job in summary['jobs'].iteritems():
                        total += 1
                        if job['State'] not in allJobStatus: allJobStatus[job['State']] = 0
                        allJobStatus[job['State']] += 1
                        if job['State'] in ['failed']:
                            failed += 1
                            resubmit = True
                if resubmit:
                    log.info('Resubmitting {0}'.format(d))
                    log.info('{0} of {1} jobs failed'.format(failed,total))
                    log.info(' '.join(['{0}: {1}'.format(state,allJobStatus[state]) for state in allowedStates if state in allJobStatus]))
                    resubmitMap[d] = crabClientResubmit.resubmit(logger,resubmitArgs)()
            except HTTPException as hte:
                log.warning("Submission for input directory {0} failed: {1}".format(d, hte.headers))
            except ClientException as cle:
                log.warning("Submission for input directory {0} failed: {1}".format(d, cle))

    for d,statMap in resubmitMap.iteritems():
        if statMap['status'] != 'SUCCESS':
            log.info('Status: {0} - {1}'.format(statMap['status'],d))
Beispiel #4
0
def resubmit_crab(args):
    '''Resubmit jobs'''
    crab_dirs = []
    if args.jobName:
        workArea = get_crab_workArea(args)
        crab_dirs += sorted(glob.glob('{0}/*'.format(workArea)))
    elif args.crabDirectories:
        for d in args.crabDirectories:
            crab_dirs += glob.glob(d)
    else:
        log.error("Shouldn't be possible to get here")

    tblogger, logger, memhandler = initLoggers()
    tblogger.setLevel(logging.WARNING)
    logger.setLevel(logging.WARNING)
    memhandler.setLevel(logging.WARNING)

    resubmitMap = {}
    for d in crab_dirs:
        if os.path.exists(d):
            statusArgs = ['--dir', d]
            resubmitArgs = ['--dir', d]
            try:
                summary = crabClientStatus.status(logger, statusArgs)()
                resubmit = False
                total = 0
                failed = 0
                if 'jobs' in summary:
                    for j, job in summary['jobs'].iteritems():
                        total += 1
                        if job['State'] in ['failed']:
                            failed += 1
                            resubmit = True
                if resubmit:
                    log.info('Resubmitting {0}'.format(d))
                    log.info('{0} of {1} jobs failed'.format(failed, total))
                    resubmitMap[d] = crabClientResubmit.resubmit(
                        logger, resubmitArgs)()
            except HTTPException as hte:
                log.warning(
                    "Submission for input directory {0} failed: {1}".format(
                        d, hte.headers))
            except ClientException as cle:
                log.warning(
                    "Submission for input directory {0} failed: {1}".format(
                        d, cle))

    for d, statMap in resubmitMap.iteritems():
        if statMap['status'] != 'SUCCESS':
            log.info('Status: {0} - {1}'.format(statMap['status'], d))
Beispiel #5
0
def resubmit_crab(args):
    '''Resubmit jobs'''
    crab_dirs = []
    if args.jobName:
        workArea = get_crab_workArea(args)
        crab_dirs += sorted(glob.glob('{0}/*'.format(workArea)))
    elif args.crabDirectories:
        for d in args.crabDirectories:
            crab_dirs += glob.glob(d)
    else:
        log.error("Shouldn't be possible to get here")

    tblogger, logger, memhandler = initLoggers()
    tblogger.setLevel(logging.WARNING)
    logger.setLevel(logging.WARNING)
    memhandler.setLevel(logging.WARNING)

    resubmitMap = {}
    for d in crab_dirs:
        if os.path.exists(d):
            statusArgs = ['--dir',d]
            resubmitArgs = ['--dir',d]
            try:
                summary = crabClientStatus.status(logger,statusArgs)()
                resubmit = False
                total = 0
                failed = 0
                if 'jobs' in summary:
                    for j,job in summary['jobs'].iteritems():
                        total += 1
                        if job['State'] in ['failed']:
                            failed += 1
                            resubmit = True
                if resubmit:
                    log.info('Resubmitting {0}'.format(d))
                    log.info('{0} of {1} jobs failed'.format(failed,total))
                    resubmitMap[d] = crabClientResubmit.resubmit(logger,resubmitArgs)()
            except HTTPException as hte:
                log.warning("Submission for input directory {0} failed: {1}".format(d, hte.headers))
            except ClientException as cle:
                log.warning("Submission for input directory {0} failed: {1}".format(d, cle))

    for d,statMap in resubmitMap.iteritems():
        if statMap['status'] != 'SUCCESS':
            log.info('Status: {0} - {1}'.format(statMap['status'],d))
Beispiel #6
0
        print ' Task ' + shortname + ' status is ' + res['status']

        if res['status'] == 'SUBMITTED' or res['status'] == 'QUEUED':
            if KILL:
                print ' Killing jobs..'
                try:
                    killobj = kill(logger, ['--dir', taskdir])
                    killobj()
                except:
                    print ' Failed to kill ' + shortname

            else:
                print ' Resubmitting potential failed jobs..'
                try:
                    resubmitobj = resubmit(logger, ['--dir', taskdir])
                    resubmitobj()
                except:
                    print ' Failed to resubmit ' + shortname

        elif res['status'] == 'COMPLETED':
            print ' Clearing.'
            shutil.rmtree(taskdir)

        elif res['status'] in ['KILLED', 'KILLFAILED', 'FAILED', 'SUBMITFAILED', 'RESUBMITFAILED']:
            print ' Obtaining the list of lumis not analyzed.'
            try:
                reportobj = report(logger, ['--dir', taskdir])
                reportobj()
            except:
                print ' Failed to fetch the lumi list.'
Beispiel #7
0
def resubmit_crab(args):
    '''Resubmit jobs'''
    if not crabLoaded:
        logging.error(
            'You must source a crab environment to submit to crab.\nsource /cvmfs/cms.cern.ch/crab3/crab.sh'
        )
        return
    crab_dirs = []
    if args.jobName:
        workArea = get_crab_workArea(args)
        crab_dirs += sorted(glob.glob('{0}/*'.format(workArea)))
    elif args.crabDirectories:
        for d in args.crabDirectories:
            crab_dirs += glob.glob(d)
    else:
        log.error("Shouldn't be possible to get here")

    tblogger, logger, memhandler = initLoggers()
    tblogger.setLevel(logging.WARNING)
    logger.setLevel(logging.WARNING)
    memhandler.setLevel(logging.WARNING)

    resubmitMap = {}
    for d in crab_dirs:
        if os.path.exists(d):
            statusArgs = ['--dir', d]
            resubmitArgs = ['--dir', d]
            try:
                summary = crabClientStatus.status(logger, statusArgs)()
                resubmit = False
                total = 0
                failed = 0
                allJobStatus = {}
                if 'jobs' in summary:
                    for j, job in summary['jobs'].iteritems():
                        total += 1
                        if job['State'] not in allJobStatus:
                            allJobStatus[job['State']] = 0
                        allJobStatus[job['State']] += 1
                        if job['State'] in ['failed']:
                            failed += 1
                            resubmit = True
                if resubmit:
                    log.info('Resubmitting {0}'.format(d))
                    log.info('{0} of {1} jobs failed'.format(failed, total))
                    log.info(' '.join([
                        '{0}: {1}'.format(state, allJobStatus[state])
                        for state in allowedStates if state in allJobStatus
                    ]))
                    resubmitMap[d] = crabClientResubmit.resubmit(
                        logger, resubmitArgs)()
            except HTTPException as hte:
                log.warning(
                    "Submission for input directory {0} failed: {1}".format(
                        d, hte.headers))
            except ClientException as cle:
                log.warning(
                    "Submission for input directory {0} failed: {1}".format(
                        d, cle))

    for d, statMap in resubmitMap.iteritems():
        if statMap['status'] != 'SUCCESS':
            log.info('Status: {0} - {1}'.format(statMap['status'], d))