def submitJob(self, jdl):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists(jdl):
fic = open(jdl, "r")
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
# Check the validity of the input JDL
jdlString = jdlString.strip()
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR('Invalid job JDL')
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob)
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
result = self.jobManager.submitJob(classAdJob.asJDL())
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn("Need to upload the proxy")
return result
def getJobDefinition(self, job, jobDef=False):
""" Retrieve JDL of the Job and return jobDef dictionary
"""
if not jobDef:
jobDef = {}
# If not jdl in jobinfo load it
if 'jdl' not in jobDef:
if self.requiredJobInfo == 'jdlOriginal':
result = self.jobDB.getJobJDL(job, original=True)
if not result['OK']:
self.log.error("No JDL for job", "%s" % job)
return S_ERROR("No JDL for job")
jobDef['jdl'] = result['Value']
if self.requiredJobInfo == 'jdl':
result = self.jobDB.getJobJDL(job)
if not result['OK']:
self.log.error("No JDL for job", "%s" % job)
return S_ERROR("No JDL for job")
jobDef['jdl'] = result['Value']
# Load the classad if needed
if 'jdl' in jobDef and 'classad' not in jobDef:
try:
classad = ClassAd(jobDef['jdl'])
except BaseException:
self.log.debug("Cannot load JDL")
return S_ERROR('Illegal Job JDL')
if not classad.isOK():
self.log.debug(
"Warning: illegal JDL for job %s, will be marked problematic"
% (job))
return S_ERROR('Illegal Job JDL')
jobDef['classad'] = classad
return S_OK(jobDef)
def insertJobInQueue(self, job, classAdJob):
""" Check individual job and add to the Task Queue eventually.
"""
jobReq = classAdJob.get_expression("JobRequirements")
classAdJobReq = ClassAd(jobReq)
jobReqDict = {}
for name in self.taskQueueDB.getSingleValueTQDefFields():
if classAdJobReq.lookupAttribute(name):
if name == 'CPUTime':
jobReqDict[name] = classAdJobReq.getAttributeInt(name)
else:
jobReqDict[name] = classAdJobReq.getAttributeString(name)
for name in self.taskQueueDB.getMultiValueTQDefFields():
if classAdJobReq.lookupAttribute(name):
jobReqDict[name] = classAdJobReq.getListFromExpression(name)
jobPriority = classAdJobReq.getAttributeInt('UserPriority')
result = self.taskQueueDB.insertJob(job, jobReqDict, jobPriority)
if not result['OK']:
self.log.error("Cannot insert job %s in task queue: %s" %
(job, result['Message']))
# Force removing the job from the TQ if it was actually inserted
result = self.taskQueueDB.deleteJob(job)
if result['OK']:
if result['Value']:
self.log.info("Job %s removed from the TQ" % job)
return S_ERROR("Cannot insert in task queue")
return S_OK()
def __getJDLParameters(self, jdl):
"""Returns a dictionary of JDL parameters.
"""
try:
parameters = {}
# print jdl
if not re.search('\[', jdl):
jdl = '[' + jdl + ']'
classAdJob = ClassAd(jdl)
paramsDict = classAdJob.contents
for param, value in paramsDict.items():
if value.strip().startswith('{'):
self.log.debug('Found list type parameter %s' % (param))
rawValues = value.replace('{',
'').replace('}', '').replace(
'"', '').split()
valueList = []
for val in rawValues:
if re.search(',$', val):
valueList.append(val[:-1])
else:
valueList.append(val)
parameters[param] = valueList
else:
parameters[param] = value.replace('"', '').replace(
'{', '"{').replace('}', '}"')
self.log.debug('Found standard parameter %s: %s' %
(param, parameters[param]))
return S_OK(parameters)
except Exception, x:
self.log.exception(lException=x)
return S_ERROR('Exception while extracting JDL parameters for job')
def test_SimpleParametricJob():
job = Job()
job.setExecutable("myExec")
job.setLogLevel("DEBUG")
parList = [1, 2, 3]
job.setParameterSequence("JOB_ID", parList, addToWorkflow=True)
inputDataList = [
["/lhcb/data/data1", "/lhcb/data/data2"],
["/lhcb/data/data3", "/lhcb/data/data4"],
["/lhcb/data/data5", "/lhcb/data/data6"],
]
job.setParameterSequence("InputData", inputDataList, addToWorkflow=True)
jdl = job._toJDL()
with open(join(dirname(__file__), "testWF.jdl")) as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd("[" + jdl + "]")
arguments = clad.getAttributeString("Arguments")
job_id = clad.getAttributeString("JOB_ID")
inputData = clad.getAttributeString("InputData")
assert job_id == "%(JOB_ID)s"
assert inputData == "%(InputData)s"
assert "jobDescription.xml" in arguments
assert "-o LogLevel=DEBUG" in arguments
assert "-p JOB_ID=%(JOB_ID)s" in arguments
assert "-p InputData=%(InputData)s" in arguments
def test_NoParameters(self):
clad = ClassAd(TEST_JDL_NO_PARAMETERS)
result = getParameterVectorLength(clad)
self.assertTrue(result['OK'])
nParam = result['Value']
self.assertTrue(nParam is None)
def test_SimpleParametricJob(self):
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
['/lhcb/data/data3', '/lhcb/data/data4'],
['/lhcb/data/data5', '/lhcb/data/data6']]
job.setParameterSequence('InputData',
inputDataList,
addToWorkflow=True)
jdl = job._toJDL()
print jdl
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
print "arguments", arguments
self.assertEqual(job_id, '%(JOB_ID)s')
self.assertEqual(inputData, '%(InputData)s')
self.assertIn('jobDescription.xml', arguments)
self.assertIn('-o LogLevel=DEBUG', arguments)
self.assertIn('-p JOB_ID=%(JOB_ID)s', arguments)
self.assertIn('-p InputData=%(InputData)s', arguments)
def getJobDefinition(self, job, jobDef=False):
"""Retrieve JDL of the Job and return jobDef dictionary"""
if not jobDef:
jobDef = {}
# If not jdl in jobinfo load it
if "jdl" not in jobDef:
if self.requiredJobInfo == "jdlOriginal":
result = self.jobDB.getJobJDL(job, original=True)
if not result["OK"]:
self.log.error("No JDL for job", "%s" % job)
return S_ERROR("No JDL for job")
jobDef["jdl"] = result["Value"]
if self.requiredJobInfo == "jdl":
result = self.jobDB.getJobJDL(job)
if not result["OK"]:
self.log.error("No JDL for job", "%s" % job)
return S_ERROR("No JDL for job")
jobDef["jdl"] = result["Value"]
# Load the classad if needed
if "jdl" in jobDef and "classad" not in jobDef:
try:
classad = ClassAd(jobDef["jdl"])
except Exception:
self.log.debug("Cannot load JDL")
return S_ERROR(JobMinorStatus.ILLEGAL_JOB_JDL)
if not classad.isOK():
self.log.debug(
"Warning: illegal JDL for job %s, will be marked problematic"
% (job))
return S_ERROR(JobMinorStatus.ILLEGAL_JOB_JDL)
jobDef["classad"] = classad
return S_OK(jobDef)
def _getJDLParameters(self, jdl):
"""Returns a dictionary of JDL parameters."""
try:
parameters = {}
# print jdl
if not re.search(r"\[", jdl):
jdl = "[" + jdl + "]"
classAdJob = ClassAd(jdl)
paramsDict = classAdJob.contents
for param, value in paramsDict.items():
if value.strip().startswith("{"):
self.log.debug("Found list type parameter %s" % (param))
rawValues = value.replace("{",
"").replace("}", "").replace(
'"', "").split()
valueList = []
for val in rawValues:
if re.search(",$", val):
valueList.append(val[:-1])
else:
valueList.append(val)
parameters[param] = valueList
else:
parameters[param] = value.replace('"', "").replace(
"{", '"{').replace("}", '}"')
self.log.debug("Found standard parameter %s: %s" %
(param, parameters[param]))
return S_OK(parameters)
except Exception as x:
self.log.exception(lException=x)
return S_ERROR("Exception while extracting JDL parameters for job")
def test_SimpleParametricJob():
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
['/lhcb/data/data3', '/lhcb/data/data4'],
['/lhcb/data/data5', '/lhcb/data/data6']]
job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)
jdl = job._toJDL()
try:
with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
assert job_id == '%(JOB_ID)s'
assert inputData == '%(InputData)s'
assert 'jobDescription.xml' in arguments
assert '-o LogLevel=DEBUG' in arguments
assert '-p JOB_ID=%(JOB_ID)s' in arguments
assert '-p InputData=%(InputData)s' in arguments
def test_SimpleProgression(self):
clad = ClassAd(TEST_JDL_SIMPLE_PROGRESSION)
nParam = getNumberOfParameters(clad)
self.assertEqual(nParam, 3)
result = generateParametricJobs(clad)
self.assert_(result['OK'])
jobDescList = result['Value']
self.assertEqual(nParam, len(jobDescList))
# Check the definition of the 2nd job
jobClassAd = ClassAd(jobDescList[1])
self.assertEqual(jobClassAd.getAttributeString('Arguments'), '3')
self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
def _saveJobJDLRequest(self, jobID, jobJDL):
"""Save job JDL local to JobAgent."""
classAdJob = ClassAd(jobJDL)
classAdJob.insertAttributeString("LocalCE", self.ceName)
jdlFileName = jobID + ".jdl"
jdlFile = open(jdlFileName, "w")
jdl = classAdJob.asJDL()
jdlFile.write(jdl)
jdlFile.close()
def test_Simple(self):
clad = ClassAd(TEST_JDL_SIMPLE)
result = getParameterVectorLength(clad)
self.assertTrue(result['OK'])
nParam = result['Value']
self.assertEqual(nParam, 3)
result = generateParametricJobs(clad)
self.assertTrue(result['OK'])
jobDescList = result['Value']
self.assertEqual(nParam, len(jobDescList))
# Check the definition of the 2nd job
jobClassAd = ClassAd(jobDescList[1])
self.assertEqual(jobClassAd.getAttributeString('Arguments'), 'b')
self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
def test_SimpleBunch(self):
clad = ClassAd(TEST_JDL_SIMPLE_BUNCH)
result = getParameterVectorLength(clad)
self.assertTrue(result["OK"])
nParam = result["Value"]
self.assertEqual(nParam, 3)
result = generateParametricJobs(clad)
self.assertTrue(result["OK"])
jobDescList = result["Value"]
self.assertEqual(nParam, len(jobDescList))
# Check the definition of the 2nd job
jobClassAd = ClassAd(jobDescList[1])
self.assertEqual(jobClassAd.getAttributeString("Arguments"), "5")
self.assertEqual(jobClassAd.getAttributeString("JobName"), "Test_1")
def _getProcessingType(self, jobID):
"""Get the Processing Type from the JDL, until it is promoted to a real Attribute"""
processingType = "unknown"
result = self.jobDB.getJobJDL(jobID, original=True)
if not result["OK"]:
return processingType
classAdJob = ClassAd(result["Value"])
if classAdJob.lookupAttribute("ProcessingType"):
processingType = classAdJob.getAttributeString("ProcessingType")
return processingType
def __processResourceDescription(self, resourceDescription):
# Check and form the resource description dictionary
resourceDict = {}
if type(resourceDescription) in StringTypes:
classAdAgent = ClassAd(resourceDescription)
if not classAdAgent.isOK():
return S_ERROR('Illegal Resource JDL')
gLogger.verbose(classAdAgent.asJDL())
for name in gTaskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute(name):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(
name)
for name in gTaskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute(name):
if name == 'SubmitPool':
resourceDict[
name] = classAdAgent.getListFromExpression(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(
name)
# Check if a JobID is requested
if classAdAgent.lookupAttribute('JobID'):
resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID')
for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject',
'VirtualOrganization'):
if classAdAgent.lookupAttribute(k):
resourceDict[k] = classAdAgent.getAttributeString(k)
else:
for name in gTaskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
for name in gTaskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key('JobID'):
resourceDict['JobID'] = resourceDescription['JobID']
for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject',
'VirtualOrganization', 'PilotReference',
'PilotInfoReportedFlag', 'PilotBenchmark',
'LHCbPlatform'):
if k in resourceDescription:
resourceDict[k] = resourceDescription[k]
return resourceDict
def __processResourceDescription(self, resourceDescription):
# Check and form the resource description dictionary
resourceDict = {}
if type(resourceDescription) in StringTypes:
classAdAgent = ClassAd(resourceDescription)
if not classAdAgent.isOK():
return S_ERROR('Illegal Resource JDL')
gLogger.verbose(classAdAgent.asJDL())
for name in gTaskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute(name):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(
name)
for name in gTaskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute(name):
resourceDict[name] = classAdAgent.getAttributeString(name)
# Check if a JobID is requested
if classAdAgent.lookupAttribute('JobID'):
resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID')
if classAdAgent.lookupAttribute('DIRACVersion'):
resourceDict['DIRACVersion'] = classAdAgent.getAttributeString(
'DIRACVersion')
if classAdAgent.lookupAttribute('VirtualOrganization'):
resourceDict[
'VirtualOrganization'] = classAdAgent.getAttributeString(
'VirtualOrganization')
else:
for name in gTaskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
for name in gTaskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key('JobID'):
resourceDict['JobID'] = resourceDescription['JobID']
if resourceDescription.has_key('DIRACVersion'):
resourceDict['DIRACVersion'] = resourceDescription[
'DIRACVersion']
if resourceDescription.has_key('VirtualOrganization'):
resourceDict['VirtualOrganization'] = resourceDescription[
'VirtualOrganization']
return resourceDict
def __saveJobJDLRequest(self, jobID, jobJDL):
"""Save job JDL local to JobAgent.
"""
classAdJob = ClassAd(jobJDL)
classAdJob.insertAttributeString('LocalCE', self.ceName)
jdlFileName = jobID + '.jdl'
jdlFile = open(jdlFileName, 'w')
jdl = classAdJob.asJDL()
jdlFile.write(jdl)
jdlFile.close()
return S_OK(jdlFileName)
def __getProcessingType(self, jobID):
""" Get the Processing Type from the JDL, until it is promoted to a real Attribute
"""
processingType = 'unknown'
result = self.jobDB.getJobJDL(jobID, original=True)
if not result['OK']:
return processingType
classAdJob = ClassAd(result['Value'])
if classAdJob.lookupAttribute('ProcessingType'):
processingType = classAdJob.getAttributeString('ProcessingType')
return processingType
def _processResourceDescription( self, resourceDescription ):
""" Check and form the resource description dictionary
resourceDescription is a ceDict coming from a JobAgent, for example.
"""
resourceDict = {}
if type( resourceDescription ) in StringTypes:
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
raise ValueError( 'Illegal Resource JDL' )
self.log.verbose( classAdAgent.asJDL() )
for name in singleValueDefFields:
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in multiValueMatchFields:
if classAdAgent.lookupAttribute( name ):
if name == 'SubmitPool':
resourceDict[name] = classAdAgent.getListFromExpression( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
if classAdAgent.lookupAttribute( k ):
resourceDict[ k ] = classAdAgent.getAttributeString( k )
else:
for name in singleValueDefFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in multiValueMatchFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ):
if k in resourceDescription:
resourceDict[ k ] = resourceDescription[ k ]
return resourceDict
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc):
job = Job()
job.setExecutable("myExec")
job.setLogLevel("DEBUG")
job.setNumberOfProcessors(proc, minProc, maxProc)
jdl = job._toJDL()
clad = ClassAd("[" + jdl + "]")
processors = clad.getAttributeInt("NumberOfProcessors")
minProcessors = clad.getAttributeInt("MinNumberOfProcessors")
maxProcessors = clad.getAttributeInt("MaxNumberOfProcessors")
assert processors == expectedProc
assert minProcessors == expectedMinProc
assert maxProcessors == expectedMaxProc
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc,
expectedMaxProc):
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
job.setNumberOfProcessors(proc, minProc, maxProc)
jdl = job._toJDL()
clad = ClassAd('[' + jdl + ']')
processors = clad.getAttributeInt('NumberOfProcessors')
minProcessors = clad.getAttributeInt('MinNumberOfProcessors')
maxProcessors = clad.getAttributeInt('MaxNumberOfProcessors')
assert processors == expectedProc
assert minProcessors == expectedMinProc
assert maxProcessors == expectedMaxProc
def submitJob(self, jdl, jobDescriptionObject=None):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists(jdl):
fic = open(jdl, "r")
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split('\n'):
if not line.strip().startswith('#'):
newJdlList.append(line)
jdlString = '\n'.join(newJdlList)
# Check the validity of the input JDL
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR('Invalid job JDL')
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject)
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
result = self.jobManager.submitJob(classAdJob.asJDL())
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn("Need to upload the proxy")
return result
def export_submitJob(self, jobDesc):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR("Can't submit using a limited proxy! (bad boy!)")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR('Failed to get job policies')
policyDict = result['Value']
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR('Job submission not authorized')
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parametric one
jobClassAd = ClassAd(jobDesc)
nParameters = getNumberOfParameters(jobClassAd)
parametricJob = False
if nParameters > 0:
parametricJob = True
result = generateParametricJobs(jobClassAd)
if not result['OK']:
return result
jobDescList = result['Value']
else:
jobDescList = [jobDesc]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner,
self.ownerDN, self.ownerGroup,
self.diracSetup)
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info('Job %s added to the JobDB for %s/%s' %
(jobID, self.ownerDN, self.ownerGroup))
gJobLoggingDB.addLoggingRecord(jobID,
result['Status'],
result['MinorStatus'],
source='JobManager')
jobIDList.append(jobID)
#Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN,
self.ownerGroup)
if 'Value' not in retVal or not retVal['Value']:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result['JobID'] = result['Value']
result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
self.__sendJobsToOptimizationMind(jobIDList)
return result
def export_submitJob(self, jobDesc):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR("Can't submit using a limited proxy! (bad boy!)")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR('Failed to get job policies')
policyDict = result['Value']
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR('Job submission not authorized')
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parameteric one
jobClassAd = ClassAd(jobDesc)
parametricJob = False
if jobClassAd.lookupAttribute('Parameters'):
parametricJob = True
if jobClassAd.isAttributeList('Parameters'):
parameterList = jobClassAd.getListFromExpression('Parameters')
else:
pStep = 0
pFactor = 1
pStart = 1
nParameters = jobClassAd.getAttributeInt('Parameters')
if not nParameters:
value = jobClassAd.get_expression('Parameters')
return S_ERROR(
'Illegal value for Parameters JDL field: %s' % value)
if jobClassAd.lookupAttribute('ParameterStart'):
value = jobClassAd.get_expression(
'ParameterStart').replace('"', '')
try:
pStart = int(value)
except:
try:
pStart = float(value)
except:
return S_ERROR(
'Illegal value for ParameterStart JDL field: %s'
% value)
if jobClassAd.lookupAttribute('ParameterStep'):
pStep = jobClassAd.getAttributeInt('ParameterStep')
if not pStep:
pStep = jobClassAd.getAttributeFloat('ParameterStep')
if not pStep:
value = jobClassAd.get_expression('ParameterStep')
return S_ERROR(
'Illegal value for ParameterStep JDL field: %s'
% value)
if jobClassAd.lookupAttribute('ParameterFactor'):
pFactor = jobClassAd.getAttributeInt('ParameterFactor')
if not pFactor:
pFactor = jobClassAd.getAttributeFloat(
'ParameterFactor')
if not pFactor:
value = jobClassAd.get_expression(
'ParameterFactor')
return S_ERROR(
'Illegal value for ParameterFactor JDL field: %s'
% value)
parameterList = list()
parameterList.append(pStart)
for i in range(nParameters - 1):
parameterList.append(parameterList[i] * pFactor + pStep)
if len(parameterList) > self.maxParametricJobs:
return S_ERROR(
'The number of parametric jobs exceeded the limit of %d' %
self.maxParametricJobs)
jobDescList = []
nParam = len(parameterList) - 1
for n, p in enumerate(parameterList):
newJobDesc = jobDesc.replace('%s', str(p)).replace(
'%n',
str(n).zfill(len(str(nParam))))
newClassAd = ClassAd(newJobDesc)
for attr in ['Parameters', 'ParameterStep', 'ParameterFactor']:
newClassAd.deleteAttribute(attr)
if type(p) == type(' ') and p.startswith('{'):
newClassAd.insertAttributeInt('Parameter', str(p))
else:
newClassAd.insertAttributeString('Parameter', str(p))
newClassAd.insertAttributeInt('ParameterNumber', n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
else:
jobDescList = [jobDesc]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner,
self.ownerDN, self.ownerGroup,
self.diracSetup)
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info('Job %s added to the JobDB for %s/%s' %
(jobID, self.ownerDN, self.ownerGroup))
gJobLoggingDB.addLoggingRecord(jobID,
result['Status'],
result['MinorStatus'],
source='JobManager')
jobIDList.append(jobID)
#Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN,
self.ownerGroup)
if 'Value' not in retVal or not retVal['Value']:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result['JobID'] = result['Value']
result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
self.__sendNewJobsToMind(jobIDList)
return result
def submitJob(self, jdl, jobDescriptionObject=None):
"""Submit one job specified by its JDL to WMS.
The JDL may actually be the desciption of a parametric job,
resulting in multiple DIRAC jobs submitted to the DIRAC WMS
"""
if os.path.exists(jdl):
with open(jdl, "r") as fic:
jdlString = fic.read()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
gLogger.debug("Submitting JDL", jdlString)
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split("\n"):
if not line.strip().startswith("#"):
newJdlList.append(line)
jdlString = "\n".join(newJdlList)
# Check the validity of the input JDL
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR(EWMSJDL, "Invalid job JDL")
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject)
if not result["OK"]:
return result
# Submit the job now and get the new job ID
result = getParameterVectorLength(classAdJob)
if not result["OK"]:
return result
nJobs = result["Value"]
result = self.jobManager.submitJob(classAdJob.asJDL())
if nJobs:
gLogger.debug("Applying transactional job submission")
# The server applies transactional bulk submission, we should confirm the jobs
if result["OK"]:
jobIDList = result["Value"]
if len(jobIDList) == nJobs:
# Confirm the submitted jobs
confirmed = False
for _attempt in range(3):
result = self.jobManager.confirmBulkSubmission(jobIDList)
if result["OK"]:
confirmed = True
break
time.sleep(1)
if not confirmed:
# The bulk submission failed, try to remove the created jobs
resultDelete = self.jobManager.removeJob(jobIDList)
error = "Job submission failed to confirm bulk transaction"
if not resultDelete["OK"]:
error += "; removal of created jobs failed"
return S_ERROR(EWMSSUBM, error)
else:
return S_ERROR(EWMSSUBM, "The number of submitted jobs does not match job description")
if result.get("requireProxyUpload"):
gLogger.warn("Need to upload the proxy")
return result
def export_submitJob(self, jobDesc):
""" Submit a job to DIRAC WMS.
The job can be a single job, or a parametric job.
If it is a parametric job, then the parameters will need to be unpacked.
:param str jobDesc: job description JDL (of a single or parametric job)
:return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK.
"""
if self.peerUsesLimitedProxy:
return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR(EWMSSUBM, 'Failed to get job policies')
policyDict = result['Value']
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR(EWMSSUBM, 'Job submission not authorized')
# jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parametric one
jobClassAd = ClassAd(jobDesc)
result = getParameterVectorLength(jobClassAd)
if not result['OK']:
return result
nJobs = result['Value']
parametricJob = False
if nJobs > 0:
# if we are here, then jobDesc was the description of a parametric job. So we start unpacking
parametricJob = True
if nJobs > self.maxParametricJobs:
return S_ERROR(EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs)
result = generateParametricJobs(jobClassAd)
if not result['OK']:
return result
jobDescList = result['Value']
else:
# if we are here, then jobDesc was the description of a single job.
jobDescList = [jobDesc]
jobIDList = []
if parametricJob:
initialStatus = 'Submitting'
initialMinorStatus = 'Bulk transaction confirmation'
else:
initialStatus = 'Received'
initialMinorStatus = 'Job accepted'
for jobDescription in jobDescList: # jobDescList because there might be a list generated by a parametric job
result = gJobDB.insertNewJobIntoDB(jobDescription,
self.owner,
self.ownerDN,
self.ownerGroup,
self.diracSetup,
initialStatus=initialStatus,
initialMinorStatus=initialMinorStatus)
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup))
gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager')
jobIDList.append(jobID)
# Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup)
if 'Value' not in retVal or not retVal['Value']:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result['JobID'] = result['Value']
result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
return result
def generateParametricJobs(jobClassAd):
""" Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute('Parameters'):
return S_OK([jobClassAd.asJDL()])
result = getParameterVectorLength(jobClassAd)
if not result['OK']:
return result
nParValues = result['Value']
if nParValues is None:
return S_ERROR(EWMSJDL,
'Can not determine the number of job parameters')
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [
'Parameters', 'ParameterStart', 'ParameterStep',
'ParameterFactor'
]:
if attribute.startswith(key):
seqID = '0' if '.' not in attribute else attribute.split(
'.')[1]
parameterDict.setdefault(seqID, {})
if key == 'Parameters':
if jobClassAd.isAttributeList(attribute):
parList = jobClassAd.getListFromExpression(attribute)
if len(parList) != nParValues:
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description')
parameterDict[seqID]['ParameterList'] = parList
else:
if attribute != "Parameters":
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description')
nPar = jobClassAd.getAttributeInt(attribute)
if nPar is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description: %s=%s'
% (attribute, value))
parameterDict[seqID]['Parameters'] = nPar
else:
value = jobClassAd.getAttributeInt(attribute)
if value is None:
value = jobClassAd.getAttributeFloat(attribute)
if value is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
'Illegal value for %s JDL field: %s' %
(attribute, value))
parameterDict[seqID][key] = value
if '0' in parameterDict and not parameterDict.get('0'):
parameterDict.pop('0')
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence(
nParValues,
parList=parameterDict[seqID].get('ParameterList', []),
parStart=parameterDict[seqID].get('ParameterStart', 1),
parStep=parameterDict[seqID].get('ParameterStep', 0),
parFactor=parameterDict[seqID].get('ParameterFactor', 1))
if not parList:
return S_ERROR(EWMSJDL, 'Inconsistent parametric job description')
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len(str(nParValues - 1))
for n in range(nParValues):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace('%n', str(n).zfill(zLength))
newClassAd = ClassAd(newJobDesc)
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute(newClassAd, attribute, seqID, str(parameter))
for seqID in parameterLists:
for attribute in [
'Parameters', 'ParameterStart', 'ParameterStep',
'ParameterFactor'
]:
if seqID == '0':
newClassAd.deleteAttribute(attribute)
else:
newClassAd.deleteAttribute('%s.%s' % (attribute, seqID))
parameter = parameterLists[seqID][n]
if seqID == '0':
attribute = 'Parameter'
else:
attribute = 'Parameter.%s' % seqID
if isinstance(parameter,
six.string_types) and parameter.startswith('{'):
newClassAd.insertAttributeInt(attribute, str(parameter))
else:
newClassAd.insertAttributeString(attribute, str(parameter))
newClassAd.insertAttributeInt('ParameterNumber', n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
return S_OK(jobDescList)
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def matchQueue(jobJDL, queueDict, fullMatch=False):
"""
Match the job description to the queue definition
:param str job: JDL job description
:param bool fullMatch: test matching on all the criteria
:param dict queueDict: queue parameters dictionary
:return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or
S_ERROR with the reason for no match
"""
# Check the job description validity
job = ClassAd(jobJDL)
if not job.isOK():
return S_ERROR("Invalid job description")
noMatchReasons = []
# Check job requirements to resource
# 1. CPUTime
cpuTime = job.getAttributeInt("CPUTime")
if not cpuTime:
cpuTime = 84600
if cpuTime > int(queueDict.get("CPUTime", 0)):
noMatchReasons.append("Job CPUTime requirement not satisfied")
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 2. Multi-value match requirements
for parameter in ["Site", "GridCE", "Platform", "JobType"]:
if parameter in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression("%ss" % parameter))
queueSet = set(fromChar(queueDict[parameter]))
if valueSet and queueSet and not valueSet.intersection(queueSet):
valueToPrint = ",".join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 3. Banned multi-value match requirements
for par in ["Site", "GridCE", "Platform", "JobType"]:
parameter = "Banned%s" % par
if par in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression("%ss" % parameter))
queueSet = set(fromChar(queueDict[par]))
if valueSet and queueSet and valueSet.issubset(queueSet):
valueToPrint = ",".join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 4. Tags
tags = set(job.getListFromExpression("Tag"))
nProc = job.getAttributeInt("NumberOfProcessors")
if nProc and nProc > 1:
tags.add("MultiProcessor")
wholeNode = job.getAttributeString("WholeNode")
if wholeNode:
tags.add("WholeNode")
queueTags = set(queueDict.get("Tag", []))
if not tags.issubset(queueTags):
noMatchReasons.append("Job Tag %s not satisfied" % ",".join(tags))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 4. MultiProcessor requirements
if nProc and nProc > int(queueDict.get("NumberOfProcessors", 1)):
noMatchReasons.append("Job NumberOfProcessors %d requirement not satisfied" % nProc)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 5. RAM
ram = job.getAttributeInt("RAM")
# If MaxRAM is not specified in the queue description, assume 2GB
if ram and ram > int(queueDict.get("MaxRAM", 2048) / 1024):
noMatchReasons.append("Job RAM %d requirement not satisfied" % ram)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# Check resource requirements to job
# 1. OwnerGroup - rare case but still
if "OwnerGroup" in queueDict:
result = getProxyInfo(disableVOMS=True)
if not result["OK"]:
return S_ERROR("No valid proxy available")
ownerGroup = result["Value"]["group"]
if ownerGroup != queueDict["OwnerGroup"]:
noMatchReasons.append("Resource OwnerGroup %s requirement not satisfied" % queueDict["OwnerGroup"])
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 2. Required tags
requiredTags = set(queueDict.get("RequiredTags", []))
if not requiredTags.issubset(tags):
noMatchReasons.append("Resource RequiredTags %s not satisfied" % ",".join(requiredTags))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 3. RunningLimit
site = queueDict["Site"]
ce = queueDict.get("GridCE")
opsHelper = Operations()
result = opsHelper.getSections("JobScheduling/RunningLimit")
if result["OK"] and site in result["Value"]:
result = opsHelper.getSections("JobScheduling/RunningLimit/%s" % site)
if result["OK"]:
for parameter in result["Value"]:
value = job.getAttributeString(parameter)
if (
value
and (
opsHelper.getValue("JobScheduling/RunningLimit/%s/%s/%s" % (site, parameter, value), 1)
or opsHelper.getValue(
"JobScheduling/RunningLimit/%s/CEs/%s/%s/%s" % (site, ce, parameter, value), 1
)
)
== 0
):
noMatchReasons.append("Resource operational %s requirement not satisfied" % parameter)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
return S_OK({"Match": not bool(noMatchReasons), "Reason": noMatchReasons})
