Пример #1
0
def parse_jobs_list(jobs_list):
    ''' parse a jobs list by first getting the status of all jobs
    '''
    from DIRAC.Interfaces.API.Dirac import Dirac
    dirac = Dirac()
    # status of all jobs
    status = dirac.getJobStatus(jobs_list)
    # parse it
    sites_dict = {}
    status_dict = copy.copy(BASE_STATUS_DIR)
    for job in jobs_list:
        site = status['Value'][int(job)]['Site']
        minstatus = status['Value'][int(job)]['MinorStatus']
        majstatus = status['Value'][int(job)]['Status']
        if majstatus not in status_dict.keys():
            DIRAC.gLogger.notice('Add %s to BASE_STATUS_DIR' % majstatus)
            DIRAC.sys.exit(1)
        status_dict[majstatus] += 1
        status_dict['Total'] += 1
        if site not in sites_dict.keys():
            if site.find('.') == -1:
                site = '    None'  # note that blank spaces are needed
            sites_dict[site] = copy.copy(BASE_STATUS_DIR)
            sites_dict[site][majstatus] = 1
            sites_dict[site]["Total"] = 1
        else:
            sites_dict[site]["Total"] += 1
            if majstatus not in sites_dict[site].keys():
                sites_dict[site][majstatus] = 1
            else:
                sites_dict[site][majstatus] += 1
    return status_dict, sites_dict
Пример #2
0
def main():
    Script.registerSwitch("f:", "File=",
                          "Get status for jobs with IDs from the file")
    Script.registerSwitch("g:", "JobGroup=",
                          "Get status for jobs in the given group")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(["JobID:    DIRAC Job ID"], mandatory=False)
    sws, args = Script.parseCommandLine(ignoreErrors=True)

    from DIRAC import exit as DIRACExit
    from DIRAC.Core.Utilities.Time import toString, date, day
    from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments

    dirac = Dirac()
    exitCode = 0

    jobs = []
    for key, value in sws:
        if key.lower() in ("f", "file"):
            if os.path.exists(value):
                jFile = open(value)
                jobs += jFile.read().split()
                jFile.close()
        elif key.lower() in ("g", "jobgroup"):
            jobDate = toString(date() - 30 * day)
            # Choose jobs no more than 30 days old
            result = dirac.selectJobs(jobGroup=value, date=jobDate)
            if not result["OK"]:
                print("Error:", result["Message"])
                DIRACExit(-1)
            jobs += result["Value"]

    if len(args) < 1 and not jobs:
        Script.showHelp(exitCode=1)

    if len(args) > 0:
        jobs += parseArguments(args)

    result = dirac.getJobStatus(jobs)
    if result["OK"]:
        for job in result["Value"]:
            print("JobID=" + str(job), end=" ")
            for status in result["Value"][job].items():
                print("%s=%s;" % status, end=" ")
            print()
    else:
        exitCode = 2
        print("ERROR: %s" % result["Message"])

    DIRACExit(exitCode)
Пример #3
0
def main():
    Script.registerSwitch("f:", "File=",
                          "Get status for jobs with IDs from the file")
    Script.registerSwitch("g:", "JobGroup=",
                          "Get status for jobs in the given group")

    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()

    from DIRAC import exit as DIRACExit
    from DIRAC.Core.Utilities.Time import toString, date, day
    from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments

    dirac = Dirac()
    exitCode = 0

    jobs = []
    for key, value in Script.getUnprocessedSwitches():
        if key.lower() in ('f', 'file'):
            if os.path.exists(value):
                jFile = open(value)
                jobs += jFile.read().split()
                jFile.close()
        elif key.lower() in ('g', 'jobgroup'):
            jobDate = toString(date() - 30 * day)
            # Choose jobs no more than 30 days old
            result = dirac.selectJobs(jobGroup=value, date=jobDate)
            if not result['OK']:
                print("Error:", result['Message'])
                DIRACExit(-1)
            jobs += result['Value']

    if len(args) < 1 and not jobs:
        Script.showHelp(exitCode=1)

    if len(args) > 0:
        jobs += parseArguments(args)

    result = dirac.getJobStatus(jobs)
    if result['OK']:
        for job in result['Value']:
            print('JobID=' + str(job), end=' ')
            for status in result['Value'][job].items():
                print('%s=%s;' % status, end=' ')
            print()
    else:
        exitCode = 2
        print("ERROR: %s" % result['Message'])

    DIRACExit(exitCode)
Пример #4
0
      jFile = open( value )
      jobs += jFile.read().split()
      jFile.close()
  elif key.lower() in ( 'g', 'jobgroup' ):
    jobDate = toString( date() - 30 * day )
    # Choose jobs no more than 30 days old
    result = dirac.selectJobs( jobGroup = value, date = jobDate )
    if not result['OK']:
      print "Error:", result['Message']
      DIRACExit( -1 )
    jobs += result['Value']

if len( args ) < 1 and not jobs:
  Script.showHelp()

if len( args ) > 0:
  jobs += parseArguments( args )

result = dirac.getJobStatus( jobs )
if result['OK']:
  for job in result['Value']:
    print 'JobID=' + str( job ),
    for status in result['Value'][job].items():
      print '%s=%s;' % status,
    print
else:
  exitCode = 2
  print "ERROR: %s" % result['Message']

DIRAC.exit( exitCode )
Пример #5
0
            jFile = open(value)
            jobs += jFile.read().split()
            jFile.close()
    elif key.lower() in ('g', 'jobgroup'):
        jobDate = toString(date() - 30 * day)
        # Choose jobs no more than 30 days old
        result = dirac.selectJobs(jobGroup=value, date=jobDate)
        if not result['OK']:
            print("Error:", result['Message'])
            DIRACExit(-1)
        jobs += result['Value']

if len(args) < 1 and not jobs:
    Script.showHelp(exitCode=1)

if len(args) > 0:
    jobs += parseArguments(args)

result = dirac.getJobStatus(jobs)
if result['OK']:
    for job in result['Value']:
        print('JobID=' + str(job), end=' ')
        for status in result['Value'][job].items():
            print('%s=%s;' % status, end=' ')
        print()
else:
    exitCode = 2
    print("ERROR: %s" % result['Message'])

DIRAC.exit(exitCode)
Пример #6
0
class CEBaseTest(TestBase):
    """
    CEBaseTest is base class for all the CE test classes. Real  CE test should
    implement its _judge method.
  """
    def __init__(self, args=None, apis=None):
        super(CEBaseTest, self).__init__(args, apis)

        self.timeout = self.args.get('timeout', 1800)
        self.vo = self.args.get('VO')
        self.testType = self.args['TestType']
        self.executable = self.args['executable']
        self.__logPath = '/opt/dirac/work/ResourceStatus/SAMTestAgent/SAM/log'
        self.__scriptPath = '/opt/dirac/pro/IHEPDIRAC/ResourceStatusSystem/SAM/sam_script'

        if 'WMSAdministrator' in self.apis:
            self.wmsAdmin = self.apis['WMSAdministrator']
        else:
            self.wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')

        if 'Dirac' in self.apis:
            self.dirac = self.apis['Dirac']
        else:
            self.dirac = Dirac()

    def doTest(self, elementDict):
        """
      submit test job to the specified ce or cloud..
    """

        elementName = elementDict['ElementName']
        elementType = elementDict['ElementType']
        vos = elementDict['VO']

        site = None
        ce = None
        if elementType == 'ComputingElement':
            ce = elementName
        if elementType == 'CLOUD':
            site = elementName

        if self.vo:
            submitVO = self.vo
        elif vos:
            submitVO = vos[0]
        else:
            submitVO = 'bes'

        submissionTime = datetime.utcnow().replace(microsecond=0)
        sendRes = self.__submit(site, ce, submitVO)
        if not sendRes['OK']:
            return sendRes
        jobID = sendRes['Value']

        result = {
            'Result': {
                'JobID': jobID,
                'VO': submitVO,
                'SubmissionTime': submissionTime
            },
            'Finish': False
        }

        return S_OK(result)

    def __submit(self, site, CE, vo):
        """
      set the job and submit.
    """

        job = Job()
        job.setName(self.testType)
        job.setJobGroup('CE-Test')
        job.setExecutable(self.executable)
        job.setInputSandbox('%s/%s' % (self.__scriptPath, self.executable))
        if site and not CE:
            job.setDestination(site)
        if CE:
            job.setDestinationCE(CE)

        LOCK.acquire()
        proxyPath = BESUtils.getProxyByVO('zhangxm', vo)
        if not proxyPath['OK']:
            LOCK.release()
            return proxyPath
        proxyPath = proxyPath['Value']
        oldProxy = os.environ.get('X509_USER_PROXY')
        os.environ['X509_USER_PROXY'] = proxyPath
        result = self.dirac.submitJob(job)
        if oldProxy is None:
            del os.environ['X509_USER_PROXY']
        else:
            os.environ['X509_USER_PROXY'] = oldProxy
        LOCK.release()

        return result

    def getTestResult(self, elementName, vo, jobID, submissionTime):
        """
      download output sandbox and judge the test status from the log file.
    """

        isFinish = False

        res = self.__getJobOutput(jobID, vo)
        if not res['OK']:
            return res
        output = res['Value']
        status = res['Status']

        resDict = {
            'CompletionTime': None,
            'Status': None,
            'Log': None,
            'ApplicationTime': None
        }
        utcNow = datetime.utcnow().replace(microsecond=0)

        if output:
            isFinish = True
            resDict['CompletionTime'] = utcNow
            log = output['Log']
            if not output['Download']:
                resDict['Status'] = 'Unknown'
                resDict['Log'] = 'Fail to download log file for job %s: %s' % (
                    jobID, log)
            else:
                resDict['Log'] = log
                resDict['Status'] = self._judge(log)
                resDict['AppliactionTime'] = self.__getAppRunningTime(log)

        else:
            if utcNow - submissionTime >= timedelta(seconds=self.timeout):
                isFinish = True
                if elementName.split('.')[0] == 'CLOUD':
                    site = elementName
                else:
                    site = BESUtils.getSiteForCE(elementName)
                jobCount = self.wmsAdmin.getSiteSummaryWeb({'Site': site}, [],
                                                           0, 0)
                if not jobCount['OK']:
                    return jobCount
                params = jobCount['Value']['ParameterNames']
                records = jobCount['Value']['Records'][0]
                run = records[params.index('Running')]
                done = records[params.index('Done')]
                if status == 'Waiting' and run == 0 and done == 0:
                    resDict['Status'] = 'Bad'
                    resDict[
                        'Log'] = 'The test job is waiting for %d seconds, but no running and done jobs at this site.' % self.timeout
                else:
                    if run != 0:
                        resDict['Status'] = 'Busy'
                        resDict[
                            'Log'] = 'Site %s is too busy to execute this test job, job status is %s' % (
                                site, status)
                    else:
                        resDict['Status'] = 'Unknown'
                        resDict[
                            'Log'] = 'Test did not complete within the timeout of %d seconds, job status is %s' % (
                                self.timeout, status)
                self.dirac.killJob(jobID)

        if not isFinish:
            return S_OK()
        else:
            return S_OK(resDict)

    def __getJobOutput(self, jobID, vo):
        status = self.dirac.getJobStatus(jobID)
        if not status['OK']:
            return status
        status = status['Value'][jobID]['Status']

        if status in ('Done', 'Failed'):
            LOCK.acquire()
            proxyPath = BESUtils.getProxyByVO('zhangxm', vo)
            if not proxyPath['OK']:
                LOCK.release()
                return proxyPath
            proxyPath = proxyPath['Value']
            oldProxy = os.environ.get('X509_USER_PROXY')
            os.environ['X509_USER_PROXY'] = proxyPath
            outputRes = self.dirac.getOutputSandbox(jobID, self.__logPath)
            if oldProxy is None:
                del os.environ['X509_USER_PROXY']
            else:
                os.environ['X509_USER_PROXY'] = oldProxy
            LOCK.release()

            if not outputRes['OK']:
                ret = S_OK({'Download': False, 'Log': outputRes['Message']})
            else:
                try:
                    logfile = open(
                        '%s/%d/Script1_CodeOutput.log' %
                        (self.__logPath, jobID), 'r')
                    log = logfile.read()
                    logfile.close()
                except IOError, e:
                    raise IOError
                os.system('rm -rf %s/%d' % (self.__logPath, jobID))
                ret = S_OK({'Download': True, 'Log': log})
        else: