def insertJobInQueue(self, job, classAdJob): """ Check individual job and add to the Task Queue eventually. """ jobReq = classAdJob.get_expression("JobRequirements") classAdJobReq = ClassAd(jobReq) jobReqDict = {} for name in self.taskQueueDB.getSingleValueTQDefFields(): if classAdJobReq.lookupAttribute(name): if name == 'CPUTime': jobReqDict[name] = classAdJobReq.getAttributeInt(name) else: jobReqDict[name] = classAdJobReq.getAttributeString(name) for name in self.taskQueueDB.getMultiValueTQDefFields(): if classAdJobReq.lookupAttribute(name): jobReqDict[name] = classAdJobReq.getListFromExpression(name) jobPriority = classAdJobReq.getAttributeInt('UserPriority') result = self.taskQueueDB.insertJob(job, jobReqDict, jobPriority) if not result['OK']: self.log.error("Cannot insert job %s in task queue: %s" % (job, result['Message'])) # Force removing the job from the TQ if it was actually inserted result = self.taskQueueDB.deleteJob(job) if result['OK']: if result['Value']: self.log.info("Job %s removed from the TQ" % job) return S_ERROR("Cannot insert in task queue") return S_OK()
def insertJobInQueue( self, job, classAdJob ): """ Check individual job and add to the Task Queue eventually. """ jobReq = classAdJob.get_expression( "JobRequirements" ) classAdJobReq = ClassAd( jobReq ) jobReqDict = {} for name in self.taskQueueDB.getSingleValueTQDefFields(): if classAdJobReq.lookupAttribute( name ): if name == 'CPUTime': jobReqDict[name] = classAdJobReq.getAttributeInt( name ) else: jobReqDict[name] = classAdJobReq.getAttributeString( name ) for name in self.taskQueueDB.getMultiValueTQDefFields(): if classAdJobReq.lookupAttribute( name ): jobReqDict[name] = classAdJobReq.getListFromExpression( name ) jobPriority = classAdJobReq.getAttributeInt( 'UserPriority' ) result = self.taskQueueDB.insertJob( job, jobReqDict, jobPriority ) if not result[ 'OK' ]: self.log.error( "Cannot insert job %s in task queue: %s" % ( job, result[ 'Message' ] ) ) # Force removing the job from the TQ if it was actually inserted result = self.taskQueueDB.deleteJob( job ) if result['OK']: if result['Value']: self.log.info( "Job %s removed from the TQ" % job ) return S_ERROR( "Cannot insert in task queue" ) return S_OK()
def __processResourceDescription(self, resourceDescription): # Check and form the resource description dictionary resourceDict = {} if type(resourceDescription) in StringTypes: classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): return S_ERROR('Illegal Resource JDL') gLogger.verbose(classAdAgent.asJDL()) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute(name): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute(name): resourceDict[name] = classAdAgent.getAttributeString(name) # Check if a JobID is requested if classAdAgent.lookupAttribute('JobID'): resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID') if classAdAgent.lookupAttribute('DIRACVersion'): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion') if classAdAgent.lookupAttribute('VirtualOrganization'): resourceDict[ 'VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization') else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key('JobID'): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key('DIRACVersion'): resourceDict['DIRACVersion'] = resourceDescription[ 'DIRACVersion'] if resourceDescription.has_key('VirtualOrganization'): resourceDict['VirtualOrganization'] = resourceDescription[ 'VirtualOrganization'] return resourceDict
def __processResourceDescription(self, resourceDescription): # Check and form the resource description dictionary resourceDict = {} if type(resourceDescription) in StringTypes: classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): return S_ERROR('Illegal Resource JDL') gLogger.verbose(classAdAgent.asJDL()) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute(name): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute(name): if name == 'SubmitPool': resourceDict[ name] = classAdAgent.getListFromExpression(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) # Check if a JobID is requested if classAdAgent.lookupAttribute('JobID'): resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID') for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization'): if classAdAgent.lookupAttribute(k): resourceDict[k] = classAdAgent.getAttributeString(k) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key('JobID'): resourceDict['JobID'] = resourceDescription['JobID'] for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotInfoReportedFlag', 'PilotBenchmark', 'LHCbPlatform'): if k in resourceDescription: resourceDict[k] = resourceDescription[k] return resourceDict
def _processResourceDescription( self, resourceDescription ): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): raise ValueError( 'Illegal Resource JDL' ) self.log.verbose( classAdAgent.asJDL() ) for name in singleValueDefFields: if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in multiValueMatchFields: if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in singleValueDefFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def _processResourceDescription( self, resourceDescription ): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if isinstance( resourceDescription, basestring ): classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): raise ValueError( 'Illegal Resource JDL' ) self.log.verbose( classAdAgent.asJDL() ) for name in singleValueDefFields: if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in multiValueMatchFields: if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in singleValueDefFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): job = Job() job.setExecutable("myExec") job.setLogLevel("DEBUG") job.setNumberOfProcessors(proc, minProc, maxProc) jdl = job._toJDL() clad = ClassAd("[" + jdl + "]") processors = clad.getAttributeInt("NumberOfProcessors") minProcessors = clad.getAttributeInt("MinNumberOfProcessors") maxProcessors = clad.getAttributeInt("MaxNumberOfProcessors") assert processors == expectedProc assert minProcessors == expectedMinProc assert maxProcessors == expectedMaxProc
def __processResourceDescription( self, resourceDescription ): # Check and form the resource description dictionary resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): return S_ERROR( 'Illegal Resource JDL' ) gLogger.verbose( classAdAgent.asJDL() ) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotInfoReportedFlag', 'PilotBenchmark' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def __processResourceDescription( self, resourceDescription ): # Check and form the resource description dictionary resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): return S_ERROR( 'Illegal Resource JDL' ) gLogger.verbose( classAdAgent.asJDL() ) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute( name ): resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) if classAdAgent.lookupAttribute( 'DIRACVersion' ): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' ) if classAdAgent.lookupAttribute( 'VirtualOrganization' ): resourceDict['VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization' ) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key( 'DIRACVersion' ): resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion'] if resourceDescription.has_key( 'VirtualOrganization' ): resourceDict['VirtualOrganization'] = resourceDescription['VirtualOrganization'] return resourceDict
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): job = Job() job.setExecutable('myExec') job.setLogLevel('DEBUG') job.setNumberOfProcessors(proc, minProc, maxProc) jdl = job._toJDL() clad = ClassAd('[' + jdl + ']') processors = clad.getAttributeInt('NumberOfProcessors') minProcessors = clad.getAttributeInt('MinNumberOfProcessors') maxProcessors = clad.getAttributeInt('MaxNumberOfProcessors') assert processors == expectedProc assert minProcessors == expectedMinProc assert maxProcessors == expectedMaxProc
def export_submitJob(self, jobDesc): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR("Can't submit using a limited proxy! (bad boy!)") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR('Failed to get job policies') policyDict = result['Value'] if not policyDict[RIGHT_SUBMIT]: return S_ERROR('Job submission not authorized') #jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parameteric one jobClassAd = ClassAd(jobDesc) parametricJob = False if jobClassAd.lookupAttribute('Parameters'): parametricJob = True if jobClassAd.isAttributeList('Parameters'): parameterList = jobClassAd.getListFromExpression('Parameters') else: pStep = 0 pFactor = 1 pStart = 1 nParameters = jobClassAd.getAttributeInt('Parameters') if not nParameters: value = jobClassAd.get_expression('Parameters') return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterStart'): value = jobClassAd.get_expression( 'ParameterStart').replace('"', '') try: pStart = int(value) except: try: pStart = float(value) except: return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterStep'): pStep = jobClassAd.getAttributeInt('ParameterStep') if not pStep: pStep = jobClassAd.getAttributeFloat('ParameterStep') if not pStep: value = jobClassAd.get_expression('ParameterStep') return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterFactor'): pFactor = jobClassAd.getAttributeInt('ParameterFactor') if not pFactor: pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor') if not pFactor: value = jobClassAd.get_expression( 'ParameterFactor') return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value) parameterList = list() parameterList.append(pStart) for i in range(nParameters - 1): parameterList.append(parameterList[i] * pFactor + pStep) if len(parameterList) > self.maxParametricJobs: return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs) jobDescList = [] nParam = len(parameterList) - 1 for n, p in enumerate(parameterList): newJobDesc = jobDesc.replace('%s', str(p)).replace( '%n', str(n).zfill(len(str(nParam)))) newClassAd = ClassAd(newJobDesc) for attr in ['Parameters', 'ParameterStep', 'ParameterFactor']: newClassAd.deleteAttribute(attr) if type(p) == type(' ') and p.startswith('{'): newClassAd.insertAttributeInt('Parameter', str(p)) else: newClassAd.insertAttributeString('Parameter', str(p)) newClassAd.insertAttributeInt('ParameterNumber', n) newJDL = newClassAd.asJDL() jobDescList.append(newJDL) else: jobDescList = [jobDesc] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup) if not result['OK']: return result jobID = result['JobID'] gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager') jobIDList.append(jobID) #Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if 'Value' not in retVal or not retVal['Value']: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired() self.__sendNewJobsToMind(jobIDList) return result
def matchQueue(jobJDL, queueDict, fullMatch=False): """ Match the job description to the queue definition :param str job: JDL job description :param bool fullMatch: test matching on all the criteria :param dict queueDict: queue parameters dictionary :return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or S_ERROR with the reason for no match """ # Check the job description validity job = ClassAd(jobJDL) if not job.isOK(): return S_ERROR("Invalid job description") noMatchReasons = [] # Check job requirements to resource # 1. CPUTime cpuTime = job.getAttributeInt("CPUTime") if not cpuTime: cpuTime = 84600 if cpuTime > int(queueDict.get("CPUTime", 0)): noMatchReasons.append("Job CPUTime requirement not satisfied") if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 2. Multi-value match requirements for parameter in ["Site", "GridCE", "Platform", "JobType"]: if parameter in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression("%ss" % parameter)) queueSet = set(fromChar(queueDict[parameter])) if valueSet and queueSet and not valueSet.intersection(queueSet): valueToPrint = ",".join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 3. Banned multi-value match requirements for par in ["Site", "GridCE", "Platform", "JobType"]: parameter = "Banned%s" % par if par in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression("%ss" % parameter)) queueSet = set(fromChar(queueDict[par])) if valueSet and queueSet and valueSet.issubset(queueSet): valueToPrint = ",".join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 4. Tags tags = set(job.getListFromExpression("Tag")) nProc = job.getAttributeInt("NumberOfProcessors") if nProc and nProc > 1: tags.add("MultiProcessor") wholeNode = job.getAttributeString("WholeNode") if wholeNode: tags.add("WholeNode") queueTags = set(queueDict.get("Tag", [])) if not tags.issubset(queueTags): noMatchReasons.append("Job Tag %s not satisfied" % ",".join(tags)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 4. MultiProcessor requirements if nProc and nProc > int(queueDict.get("NumberOfProcessors", 1)): noMatchReasons.append("Job NumberOfProcessors %d requirement not satisfied" % nProc) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 5. RAM ram = job.getAttributeInt("RAM") # If MaxRAM is not specified in the queue description, assume 2GB if ram and ram > int(queueDict.get("MaxRAM", 2048) / 1024): noMatchReasons.append("Job RAM %d requirement not satisfied" % ram) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # Check resource requirements to job # 1. OwnerGroup - rare case but still if "OwnerGroup" in queueDict: result = getProxyInfo(disableVOMS=True) if not result["OK"]: return S_ERROR("No valid proxy available") ownerGroup = result["Value"]["group"] if ownerGroup != queueDict["OwnerGroup"]: noMatchReasons.append("Resource OwnerGroup %s requirement not satisfied" % queueDict["OwnerGroup"]) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 2. Required tags requiredTags = set(queueDict.get("RequiredTags", [])) if not requiredTags.issubset(tags): noMatchReasons.append("Resource RequiredTags %s not satisfied" % ",".join(requiredTags)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 3. RunningLimit site = queueDict["Site"] ce = queueDict.get("GridCE") opsHelper = Operations() result = opsHelper.getSections("JobScheduling/RunningLimit") if result["OK"] and site in result["Value"]: result = opsHelper.getSections("JobScheduling/RunningLimit/%s" % site) if result["OK"]: for parameter in result["Value"]: value = job.getAttributeString(parameter) if ( value and ( opsHelper.getValue("JobScheduling/RunningLimit/%s/%s/%s" % (site, parameter, value), 1) or opsHelper.getValue( "JobScheduling/RunningLimit/%s/CEs/%s/%s/%s" % (site, ce, parameter, value), 1 ) ) == 0 ): noMatchReasons.append("Resource operational %s requirement not satisfied" % parameter) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) return S_OK({"Match": not bool(noMatchReasons), "Reason": noMatchReasons})
def _processResourceDescription(self, resourceDescription): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if isinstance(resourceDescription, basestring): classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): raise ValueError('Illegal Resource JDL') self.log.verbose(classAdAgent.asJDL()) for name in singleValueDefFields: if classAdAgent.lookupAttribute(name): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) for name in multiValueMatchFields: if classAdAgent.lookupAttribute(name): if name == 'SubmitPool': resourceDict[ name] = classAdAgent.getListFromExpression(name) else: resourceDict[name] = classAdAgent.getAttributeString( name) # Check if a JobID is requested if classAdAgent.lookupAttribute('JobID'): resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID') for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization'): if classAdAgent.lookupAttribute(k): resourceDict[k] = classAdAgent.getAttributeString(k) else: for name in singleValueDefFields: if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if 'JobID' in resourceDescription: resourceDict['JobID'] = resourceDescription['JobID'] # Convert MaxRAM and NumberOfCores parameters into a list of tags maxRAM = resourceDescription.get('MaxRAM') nCores = resourceDescription.get('NumberOfProcessors') for param, key in [(maxRAM, 'GB'), (nCores, 'Cores')]: if param: try: intValue = int(param) / 1000 if intValue <= 128: paramList = range(1, intValue + 1) paramTags = [ '%d%s' % (par, key) for par in paramList ] resourceDict.setdefault("Tag", []).extend(paramTags) except ValueError: pass if 'Tag' in resourceDict: resourceDict['Tag'] = list(set(resourceDict['Tag'])) for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag'): if k in resourceDescription: resourceDict[k] = resourceDescription[k] return resourceDict
def _processResourceDescription( self, resourceDescription ): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if isinstance( resourceDescription, basestring ): classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): raise ValueError( 'Illegal Resource JDL' ) self.log.verbose( classAdAgent.asJDL() ) for name in singleValueDefFields: if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in multiValueMatchFields: if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in singleValueDefFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if 'JobID' in resourceDescription: resourceDict['JobID'] = resourceDescription['JobID'] # Convert MaxRAM and NumberOfCores parameters into a list of tags maxRAM = resourceDescription.get( 'MaxRAM' ) nCores = resourceDescription.get( 'NumberOfProcessors' ) for param, key in [ ( maxRAM, 'GB' ), ( nCores, 'Cores' ) ]: if param: try: intValue = int( param )/1000 if intValue <= 128: paramList = range( 1, intValue + 1 ) paramTags = [ '%d%s' % ( par, key ) for par in paramList ] resourceDict.setdefault( "Tag", [] ).extend( paramTags ) except ValueError: pass if 'Tag' in resourceDict: resourceDict['Tag'] = list( set( resourceDict['Tag'] ) ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict
def selectJob(self, resourceDescription): """ Main job selection function to find the highest priority job matching the resource capacity """ startTime = time.time() # Check and form the resource description dictionary resourceDict = {} if type(resourceDescription) in StringTypes: classAdAgent = ClassAd(resourceDescription) if not classAdAgent.isOK(): return S_ERROR("Illegal Resource JDL") gLogger.verbose(classAdAgent.asJDL()) for name in taskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute(name): if name == "CPUTime": resourceDict[name] = classAdAgent.getAttributeInt(name) else: resourceDict[name] = classAdAgent.getAttributeString(name) for name in taskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute(name): resourceDict[name] = classAdAgent.getAttributeString(name) # Check if a JobID is requested if classAdAgent.lookupAttribute("JobID"): resourceDict["JobID"] = classAdAgent.getAttributeInt("JobID") if classAdAgent.lookupAttribute("DIRACVersion"): resourceDict["DIRACVersion"] = classAdAgent.getAttributeString("DIRACVersion") else: for name in taskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] for name in taskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key(name): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key("JobID"): resourceDict["JobID"] = resourceDescription["JobID"] if resourceDescription.has_key("DIRACVersion"): resourceDict["DIRACVersion"] = resourceDescription["DIRACVersion"] # Check the pilot DIRAC version if self.checkPilotVersion: if "DIRACVersion" in resourceDict: if self.pilotVersion and resourceDict["DIRACVersion"] != self.pilotVersion: return S_ERROR( "Pilot version does not match the production version %s:%s" % (resourceDict["DIRACVersion"], self.pilotVersion) ) # Get common site mask and check the agent site result = jobDB.getSiteMask(siteState="Active") if result["OK"]: maskList = result["Value"] else: return S_ERROR("Internal error: can not get site mask") if not "Site" in resourceDict: return S_ERROR("Missing Site Name in Resource JDL") siteName = resourceDict["Site"] if resourceDict["Site"] not in maskList: if "GridCE" in resourceDict: del resourceDict["Site"] else: return S_ERROR("Site not in mask and GridCE not specified") resourceDict["Setup"] = self.serviceInfoDict["clientSetup"] if DEBUG: print "Resource description:" for k, v in resourceDict.items(): print k.rjust(20), v # Check if Job Limits are imposed onto the site extraConditions = {} if self.siteJobLimits: result = self.getExtraConditions(siteName) if result["OK"]: extraConditions = result["Value"] if extraConditions: gLogger.info("Job Limits for site %s are: %s" % (siteName, str(extraConditions))) result = taskQueueDB.matchAndGetJob(resourceDict, extraConditions=extraConditions) if DEBUG: print result if not result["OK"]: return result result = result["Value"] if not result["matchFound"]: return S_ERROR("No match found") jobID = result["jobId"] resAtt = jobDB.getJobAttributes(jobID, ["OwnerDN", "OwnerGroup", "Status"]) if not resAtt["OK"]: return S_ERROR("Could not retrieve job attributes") if not resAtt["Value"]: return S_ERROR("No attributes returned for job") if not resAtt["Value"]["Status"] == "Waiting": gLogger.error("Job %s matched by the TQ is not in Waiting state" % str(jobID)) result = taskQueueDB.deleteJob(jobID) result = jobDB.setJobStatus(jobID, status="Matched", minor="Assigned") result = jobLoggingDB.addLoggingRecord(jobID, status="Matched", minor="Assigned", source="Matcher") result = jobDB.getJobJDL(jobID) if not result["OK"]: return S_ERROR("Failed to get the job JDL") resultDict = {} resultDict["JDL"] = result["Value"] resultDict["JobID"] = jobID matchTime = time.time() - startTime gLogger.info("Match time: [%s]" % str(matchTime)) gMonitor.addMark("matchTime", matchTime) # Get some extra stuff into the response returned resOpt = jobDB.getJobOptParameters(jobID) if resOpt["OK"]: for key, value in resOpt["Value"].items(): resultDict[key] = value resAtt = jobDB.getJobAttributes(jobID, ["OwnerDN", "OwnerGroup"]) if not resAtt["OK"]: return S_ERROR("Could not retrieve job attributes") if not resAtt["Value"]: return S_ERROR("No attributes returned for job") resultDict["DN"] = resAtt["Value"]["OwnerDN"] resultDict["Group"] = resAtt["Value"]["OwnerGroup"] return S_OK(resultDict)
def export_submitJob( self, jobDesc ): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR( "Can't submit using a limited proxy! (bad boy!)" ) # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR( 'Failed to get job policies' ) policyDict = result['Value'] if not policyDict[ RIGHT_SUBMIT ]: return S_ERROR( 'Job submission not authorized' ) #jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parameteric one jobClassAd = ClassAd( jobDesc ) parametricJob = False if jobClassAd.lookupAttribute( 'Parameters' ): parametricJob = True if jobClassAd.isAttributeList( 'Parameters' ): parameterList = jobClassAd.getListFromExpression( 'Parameters' ) else: pStep = 0 pFactor = 1 pStart = 1 nParameters = jobClassAd.getAttributeInt( 'Parameters' ) if not nParameters: value = jobClassAd.get_expression( 'Parameters' ) return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value ) if jobClassAd.lookupAttribute( 'ParameterStart' ): value = jobClassAd.get_expression( 'ParameterStart' ).replace( '"', '' ) try: pStart = int( value ) except: try: pStart = float( value ) except: return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value ) if jobClassAd.lookupAttribute( 'ParameterStep' ): pStep = jobClassAd.getAttributeInt( 'ParameterStep' ) if not pStep: pStep = jobClassAd.getAttributeFloat( 'ParameterStep' ) if not pStep: value = jobClassAd.get_expression( 'ParameterStep' ) return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value ) if jobClassAd.lookupAttribute( 'ParameterFactor' ): pFactor = jobClassAd.getAttributeInt( 'ParameterFactor' ) if not pFactor: pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor' ) if not pFactor: value = jobClassAd.get_expression( 'ParameterFactor' ) return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value ) parameterList = list() parameterList.append( pStart ) for i in range( nParameters - 1 ): parameterList.append( parameterList[i] * pFactor + pStep ) if len( parameterList ) > self.maxParametricJobs: return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs ) jobDescList = [] nParam = len(parameterList) - 1 for n,p in enumerate(parameterList): newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n).zfill(len(str(nParam)))) newClassAd = ClassAd(newJobDesc) for attr in ['Parameters','ParameterStep','ParameterFactor']: newClassAd.deleteAttribute(attr) if type( p ) == type ( ' ' ) and p.startswith('{'): newClassAd.insertAttributeInt( 'Parameter',str(p) ) else: newClassAd.insertAttributeString( 'Parameter', str( p ) ) newClassAd.insertAttributeInt( 'ParameterNumber', n ) newJDL = newClassAd.asJDL() jobDescList.append( newJDL ) else: jobDescList = [ jobDesc ] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup ) if not result['OK']: return result jobID = result['JobID'] gLogger.info( 'Job %s added to the JobDB for %s/%s' % ( jobID, self.ownerDN, self.ownerGroup ) ) gJobLoggingDB.addLoggingRecord( jobID, result['Status'], result['MinorStatus'], source = 'JobManager' ) jobIDList.append( jobID ) #Set persistency flag retVal = gProxyManager.getUserPersistence( self.ownerDN, self.ownerGroup ) if 'Value' not in retVal or not retVal[ 'Value' ]: gProxyManager.setPersistency( self.ownerDN, self.ownerGroup, True ) if parametricJob: result = S_OK( jobIDList ) else: result = S_OK( jobIDList[0] ) result['JobID'] = result['Value'] result[ 'requireProxyUpload' ] = self.__checkIfProxyUploadIsRequired() self.__sendNewJobsToMind( jobIDList ) return result
def selectJob( self, resourceDescription ): """ Main job selection function to find the highest priority job matching the resource capacity """ startTime = time.time() # Check and form the resource description dictionary resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): return S_ERROR( 'Illegal Resource JDL' ) gLogger.verbose( classAdAgent.asJDL() ) for name in taskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in taskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute( name ): resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) if classAdAgent.lookupAttribute( 'DIRACVersion' ): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' ) else: for name in taskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in taskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key( 'DIRACVersion' ): resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion'] # Check the pilot DIRAC version if self.checkPilotVersion: if 'DIRACVersion' in resourceDict: if self.pilotVersion and resourceDict['DIRACVersion'] != self.pilotVersion: return S_ERROR( 'Pilot version does not match the production version %s:%s' % \ ( resourceDict['DIRACVersion'], self.pilotVersion ) ) # Get common site mask and check the agent site result = jobDB.getSiteMask( siteState = 'Active' ) if result['OK']: maskList = result['Value'] else: return S_ERROR( 'Internal error: can not get site mask' ) if not 'Site' in resourceDict: return S_ERROR( 'Missing Site Name in Resource JDL' ) siteName = resourceDict['Site'] if resourceDict['Site'] not in maskList: if 'GridCE' in resourceDict: del resourceDict['Site'] else: return S_ERROR( 'Site not in mask and GridCE not specified' ) resourceDict['Setup'] = self.serviceInfoDict['clientSetup'] if DEBUG: print "Resource description:" for k, v in resourceDict.items(): print k.rjust( 20 ), v # Check if Job Limits are imposed onto the site extraConditions = {} if self.siteJobLimits: result = self.getExtraConditions( siteName ) if result['OK']: extraConditions = result['Value'] if extraConditions: gLogger.info( 'Job Limits for site %s are: %s' % ( siteName, str( extraConditions ) ) ) result = taskQueueDB.matchAndGetJob( resourceDict, extraConditions = extraConditions ) if DEBUG: print result if not result['OK']: return result result = result['Value'] if not result['matchFound']: return S_ERROR( 'No match found' ) jobID = result['jobId'] resAtt = jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] ) if not resAtt['OK']: return S_ERROR( 'Could not retrieve job attributes' ) if not resAtt['Value']: return S_ERROR( 'No attributes returned for job' ) if not resAtt['Value']['Status'] == 'Waiting': gLogger.error( 'Job %s matched by the TQ is not in Waiting state' % str( jobID ) ) result = taskQueueDB.deleteJob( jobID ) result = jobDB.setJobStatus( jobID, status = 'Matched', minor = 'Assigned' ) result = jobLoggingDB.addLoggingRecord( jobID, status = 'Matched', minor = 'Assigned', source = 'Matcher' ) result = jobDB.getJobJDL( jobID ) if not result['OK']: return S_ERROR( 'Failed to get the job JDL' ) resultDict = {} resultDict['JDL'] = result['Value'] resultDict['JobID'] = jobID matchTime = time.time() - startTime gLogger.info( "Match time: [%s]" % str( matchTime ) ) gMonitor.addMark( "matchTime", matchTime ) # Get some extra stuff into the response returned resOpt = jobDB.getJobOptParameters( jobID ) if resOpt['OK']: for key, value in resOpt['Value'].items(): resultDict[key] = value resAtt = jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] ) if not resAtt['OK']: return S_ERROR( 'Could not retrieve job attributes' ) if not resAtt['Value']: return S_ERROR( 'No attributes returned for job' ) resultDict['DN'] = resAtt['Value']['OwnerDN'] resultDict['Group'] = resAtt['Value']['OwnerGroup'] return S_OK( resultDict )
def matchQueue(jobJDL, queueDict, fullMatch=False): """ Match the job description to the queue definition :param str job: JDL job description :param bool fullMatch: test matching on all the criteria :param dict queueDict: queue parameters dictionary :return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or S_ERROR with the reason for no match """ # Check the job description validity job = ClassAd(jobJDL) if not job.isOK(): return S_ERROR('Invalid job description') noMatchReasons = [] # Check job requirements to resource # 1. CPUTime cpuTime = job.getAttributeInt('CPUTime') if not cpuTime: cpuTime = 84600 if cpuTime > queueDict.get('CPUTime', 0.): noMatchReasons.append('Job CPUTime requirement not satisfied') if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 2. Multi-value match requirements for parameter in [ 'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType', 'SubmitPool', 'JobType' ]: if parameter in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression('%ss' % parameter)) queueSet = set(fromChar(queueDict[parameter])) if valueSet and queueSet and not valueSet.intersection(queueSet): valueToPrint = ','.join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append('Job %s %s requirement not satisfied' % (parameter, valueToPrint)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 3. Banned multi-value match requirements for par in [ 'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType', 'SubmitPool', 'JobType' ]: parameter = "Banned%s" % par if par in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression('%ss' % parameter)) queueSet = set(fromChar(queueDict[par])) if valueSet and queueSet and valueSet.issubset(queueSet): valueToPrint = ','.join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append('Job %s %s requirement not satisfied' % (parameter, valueToPrint)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 4. Tags tags = set(job.getListFromExpression('Tag')) nProc = job.getAttributeInt('NumberOfProcessors') if nProc and nProc > 1: tags.add('MultiProcessor') wholeNode = job.getAttributeString('WholeNode') if wholeNode: tags.add('WholeNode') queueTags = set(queueDict.get('Tags', [])) if not tags.issubset(queueTags): noMatchReasons.append('Job Tag %s not satisfied' % ','.join(tags)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 4. MultiProcessor requirements if nProc and nProc > int(queueDict.get('NumberOfProcessors', 1)): noMatchReasons.append( 'Job NumberOfProcessors %d requirement not satisfied' % nProc) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 5. RAM ram = job.getAttributeInt('RAM') # If MaxRAM is not specified in the queue description, assume 2GB if ram and ram > int(queueDict.get('MaxRAM', 2048)) / 1024: noMatchReasons.append('Job RAM %d requirement not satisfied' % ram) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # Check resource requirements to job # 1. OwnerGroup - rare case but still if "OwnerGroup" in queueDict: result = getProxyInfo(disableVOMS=True) if not result['OK']: return S_ERROR('No valid proxy available') ownerGroup = result['Value']['group'] if ownerGroup != queueDict['OwnerGroup']: noMatchReasons.append( 'Resource OwnerGroup %s requirement not satisfied' % queueDict['OwnerGroup']) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 2. Required tags requiredTags = set(queueDict.get('RequiredTags', [])) if not requiredTags.issubset(tags): noMatchReasons.append('Resource RequiredTags %s not satisfied' % ','.join(requiredTags)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 3. RunningLimit site = queueDict['Site'] opsHelper = Operations() result = opsHelper.getSections('JobScheduling/RunningLimit') if result['OK'] and site in result['Value']: result = opsHelper.getSections('JobScheduling/RunningLimit/%s' % site) if result['OK']: for parameter in result['Value']: value = job.getAttributeString(parameter) if value and opsHelper.getValue( 'JobScheduling/RunningLimit/%s/%s/%s' % (site, parameter, value), 1) == 0: noMatchReasons.append( 'Resource operational %s requirement not satisfied' % parameter) if not fullMatch: return S_OK({ 'Match': False, 'Reason': noMatchReasons[0] }) return S_OK({'Match': not bool(noMatchReasons), 'Reason': noMatchReasons})
def selectJob( self, resourceDescription ): """ Main job selection function to find the highest priority job matching the resource capacity """ startTime = time.time() # Check and form the resource description dictionary resourceDict = {} if type( resourceDescription ) in StringTypes: classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): return S_ERROR( 'Illegal Resource JDL' ) gLogger.verbose( classAdAgent.asJDL() ) for name in gTaskQueueDB.getSingleValueTQDefFields(): if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in gTaskQueueDB.getMultiValueMatchFields(): if classAdAgent.lookupAttribute( name ): resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) if classAdAgent.lookupAttribute( 'DIRACVersion' ): resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' ) if classAdAgent.lookupAttribute( 'VirtualOrganization' ): resourceDict['VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization' ) else: for name in gTaskQueueDB.getSingleValueTQDefFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in gTaskQueueDB.getMultiValueMatchFields(): if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] if resourceDescription.has_key( 'DIRACVersion' ): resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion'] if resourceDescription.has_key( 'VirtualOrganization' ): resourceDict['VirtualOrganization'] = resourceDescription['VirtualOrganization'] # Check the pilot DIRAC version if self.checkPilotVersion: if not 'DIRACVersion' in resourceDict: return S_ERROR( 'Version check requested and not provided by Pilot' ) # Check if the matching Request provides a VirtualOrganization if 'VirtualOrganization' in resourceDict: voName = resourceDict['VirtualOrganization'] # Check if the matching Request provides an OwnerGroup elif 'OwnerGroup' in resourceDict: voName = getVOForGroup( resourceDict['OwnerGroup'] ) # else take the default VirtualOrganization for the installation else: voName = getVOForGroup( '' ) self.pilotVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( voName, self.setup ), '' ) if self.pilotVersion and resourceDict['DIRACVersion'] != self.pilotVersion: return S_ERROR( 'Pilot version does not match the production version %s:%s' % \ ( resourceDict['DIRACVersion'], self.pilotVersion ) ) # Get common site mask and check the agent site result = gJobDB.getSiteMask( siteState = 'Active' ) if result['OK']: maskList = result['Value'] else: return S_ERROR( 'Internal error: can not get site mask' ) if not 'Site' in resourceDict: return S_ERROR( 'Missing Site Name in Resource JDL' ) siteName = resourceDict['Site'] if resourceDict['Site'] not in maskList: if 'GridCE' in resourceDict: del resourceDict['Site'] else: return S_ERROR( 'Site not in mask and GridCE not specified' ) resourceDict['Setup'] = self.serviceInfoDict['clientSetup'] if DEBUG: print "Resource description:" for key, value in resourceDict.items(): print key.rjust( 20 ), value # Check if Job Limits are imposed onto the site extraConditions = {} if self.siteJobLimits: result = self.getExtraConditions( siteName ) if result['OK']: extraConditions = result['Value'] if extraConditions: gLogger.info( 'Job Limits for site %s are: %s' % ( siteName, str( extraConditions ) ) ) result = gTaskQueueDB.matchAndGetJob( resourceDict, extraConditions = extraConditions ) if DEBUG: print result if not result['OK']: return result result = result['Value'] if not result['matchFound']: return S_ERROR( 'No match found' ) jobID = result['jobId'] resAtt = gJobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] ) if not resAtt['OK']: return S_ERROR( 'Could not retrieve job attributes' ) if not resAtt['Value']: return S_ERROR( 'No attributes returned for job' ) if not resAtt['Value']['Status'] == 'Waiting': gLogger.error( 'Job %s matched by the TQ is not in Waiting state' % str( jobID ) ) result = gTaskQueueDB.deleteJob( jobID ) result = gJobDB.setJobStatus( jobID, status = 'Matched', minor = 'Assigned' ) result = gJobLoggingDB.addLoggingRecord( jobID, status = 'Matched', minor = 'Assigned', source = 'Matcher' ) result = gJobDB.getJobJDL( jobID ) if not result['OK']: return S_ERROR( 'Failed to get the job JDL' ) resultDict = {} resultDict['JDL'] = result['Value'] resultDict['JobID'] = jobID matchTime = time.time() - startTime gLogger.info( "Match time: [%s]" % str( matchTime ) ) gMonitor.addMark( "matchTime", matchTime ) # Get some extra stuff into the response returned resOpt = gJobDB.getJobOptParameters( jobID ) if resOpt['OK']: for key, value in resOpt['Value'].items(): resultDict[key] = value resAtt = gJobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] ) if not resAtt['OK']: return S_ERROR( 'Could not retrieve job attributes' ) if not resAtt['Value']: return S_ERROR( 'No attributes returned for job' ) resultDict['DN'] = resAtt['Value']['OwnerDN'] resultDict['Group'] = resAtt['Value']['OwnerGroup'] return S_OK( resultDict )
def export_submitJob( self, jobDesc ): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR( "Can't submit using a limited proxy! (bad boy!)" ) # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR( 'Failed to get job policies' ) policyDict = result['Value'] if not policyDict[ RIGHT_SUBMIT ]: return S_ERROR('Job submission not authorized') #jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parameteric one jobClassAd = ClassAd(jobDesc) parametricJob = False if jobClassAd.lookupAttribute('Parameters'): parametricJob = True if jobClassAd.isAttributeList('Parameters'): parameterList = jobClassAd.getListFromExpression('Parameters') else: nParameters = jobClassAd.getAttributeInt('Parameters') if not nParameters: value = jobClassAd.get_expression('Parameters') return S_ERROR('Illegal value for Parameters JDL field: %s' % value) if jobClassAd.lookupAttribute('ParameterStart'): pStart = jobClassAd.getAttributeInt('ParameterStart') else: return S_ERROR('Missing JDL field ParameterStart') if jobClassAd.lookupAttribute('ParameterStep'): pStep = jobClassAd.getAttributeInt('ParameterStep') if not pStep: value = jobClassAd.get_expression('ParameterStep') return S_ERROR('Illegal value for ParameterStep JDL field: %s' % value) else: return S_ERROR('Missing JDL field ParameterStep') parameterList = list( range(pStart,pStart+pStep*nParameters,pStep) ) if len(parameterList) > MAX_PARAMETRIC_JOBS: return S_ERROR('The number of parametric jobs exceeded the limit of %d' % MAX_PARAMETRIC_JOBS ) jobDescList = [] for p in parameterList: jobDescList.append( jobDesc.replace('%s',str(p)) ) else: jobDescList = [ jobDesc ] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup ) if not result['OK']: return result jobID = result['JobID'] gLogger.info( 'Job %s added to the JobDB for %s/%s' % ( jobID, self.ownerDN, self.ownerGroup ) ) gJobLoggingDB.addLoggingRecord( jobID, result['Status'], result['MinorStatus'], source = 'JobManager' ) jobIDList.append(jobID) #Set persistency flag retVal = gProxyManager.getUserPersistence( self.ownerDN, self.ownerGroup ) if 'Value' not in retVal or not retVal[ 'Value' ]: gProxyManager.setPersistency( self.ownerDN, self.ownerGroup, True ) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result[ 'requireProxyUpload' ] = self.__checkIfProxyUploadIsRequired() return result