def submitJob( self, jdl ):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists( jdl ):
fic = open ( jdl, "r" )
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
# Check the validity of the input JDL
jdlString = jdlString.strip()
if jdlString.find( "[" ) != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd( jdlString )
if not classAdJob.isOK():
return S_ERROR( 'Invalid job JDL' )
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox( classAdJob )
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
result = self.jobManager.submitJob( classAdJob.asJDL() )
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn( "Need to upload the proxy" )
return result
def submitJob(self, jdl):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists(jdl):
fic = open(jdl, "r")
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
# Check the validity of the input JDL
jdlString = jdlString.strip()
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR('Invalid job JDL')
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob)
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
result = self.jobManager.submitJob(classAdJob.asJDL())
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn("Need to upload the proxy")
return result
def _saveJobJDLRequest(self, jobID, jobJDL):
"""Save job JDL local to JobAgent."""
classAdJob = ClassAd(jobJDL)
classAdJob.insertAttributeString("LocalCE", self.ceName)
jdlFileName = jobID + ".jdl"
jdlFile = open(jdlFileName, "w")
jdl = classAdJob.asJDL()
jdlFile.write(jdl)
jdlFile.close()
def __saveJobJDLRequest( self, jobID, jobJDL ): """Save job JDL local to JobAgent. """ classAdJob = ClassAd( jobJDL ) classAdJob.insertAttributeString( 'LocalCE', self.ceName ) jdlFileName = jobID + '.jdl' jdlFile = open( jdlFileName, 'w' ) jdl = classAdJob.asJDL() jdlFile.write( jdl ) jdlFile.close()
def __processResourceDescription(self, resourceDescription):
# Check and form the resource description dictionary
resourceDict = {}
if type(resourceDescription) in StringTypes:
classAdAgent = ClassAd(resourceDescription)
if not classAdAgent.isOK():
return S_ERROR('Illegal Resource JDL')
gLogger.verbose(classAdAgent.asJDL())
for name in gTaskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute(name):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(
name)
for name in gTaskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute(name):
if name == 'SubmitPool':
resourceDict[
name] = classAdAgent.getListFromExpression(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(
name)
# Check if a JobID is requested
if classAdAgent.lookupAttribute('JobID'):
resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID')
for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject',
'VirtualOrganization'):
if classAdAgent.lookupAttribute(k):
resourceDict[k] = classAdAgent.getAttributeString(k)
else:
for name in gTaskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
for name in gTaskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key('JobID'):
resourceDict['JobID'] = resourceDescription['JobID']
for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject',
'VirtualOrganization', 'PilotReference',
'PilotInfoReportedFlag', 'PilotBenchmark',
'LHCbPlatform'):
if k in resourceDescription:
resourceDict[k] = resourceDescription[k]
return resourceDict
def __processResourceDescription(self, resourceDescription):
# Check and form the resource description dictionary
resourceDict = {}
if type(resourceDescription) in StringTypes:
classAdAgent = ClassAd(resourceDescription)
if not classAdAgent.isOK():
return S_ERROR('Illegal Resource JDL')
gLogger.verbose(classAdAgent.asJDL())
for name in gTaskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute(name):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(
name)
for name in gTaskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute(name):
resourceDict[name] = classAdAgent.getAttributeString(name)
# Check if a JobID is requested
if classAdAgent.lookupAttribute('JobID'):
resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID')
if classAdAgent.lookupAttribute('DIRACVersion'):
resourceDict['DIRACVersion'] = classAdAgent.getAttributeString(
'DIRACVersion')
if classAdAgent.lookupAttribute('VirtualOrganization'):
resourceDict[
'VirtualOrganization'] = classAdAgent.getAttributeString(
'VirtualOrganization')
else:
for name in gTaskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
for name in gTaskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key('JobID'):
resourceDict['JobID'] = resourceDescription['JobID']
if resourceDescription.has_key('DIRACVersion'):
resourceDict['DIRACVersion'] = resourceDescription[
'DIRACVersion']
if resourceDescription.has_key('VirtualOrganization'):
resourceDict['VirtualOrganization'] = resourceDescription[
'VirtualOrganization']
return resourceDict
def __saveJobJDLRequest(self, jobID, jobJDL):
"""Save job JDL local to JobAgent.
"""
classAdJob = ClassAd(jobJDL)
classAdJob.insertAttributeString("LocalCE", self.ceName)
jdlFileName = jobID + ".jdl"
jdlFile = open(jdlFileName, "w")
jdl = classAdJob.asJDL()
jdlFile.write(jdl)
jdlFile.close()
return S_OK(jdlFileName)
def __saveJobJDLRequest(self, jobID, jobJDL):
"""Save job JDL local to JobAgent.
"""
classAdJob = ClassAd(jobJDL)
classAdJob.insertAttributeString('LocalCE', self.ceName)
jdlFileName = jobID + '.jdl'
jdlFile = open(jdlFileName, 'w')
jdl = classAdJob.asJDL()
jdlFile.write(jdl)
jdlFile.close()
return S_OK(jdlFileName)
def _processResourceDescription( self, resourceDescription ):
""" Check and form the resource description dictionary
resourceDescription is a ceDict coming from a JobAgent, for example.
"""
resourceDict = {}
if type( resourceDescription ) in StringTypes:
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
raise ValueError( 'Illegal Resource JDL' )
self.log.verbose( classAdAgent.asJDL() )
for name in singleValueDefFields:
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in multiValueMatchFields:
if classAdAgent.lookupAttribute( name ):
if name == 'SubmitPool':
resourceDict[name] = classAdAgent.getListFromExpression( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
if classAdAgent.lookupAttribute( k ):
resourceDict[ k ] = classAdAgent.getAttributeString( k )
else:
for name in singleValueDefFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in multiValueMatchFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ):
if k in resourceDescription:
resourceDict[ k ] = resourceDescription[ k ]
return resourceDict
def _processResourceDescription( self, resourceDescription ):
""" Check and form the resource description dictionary
resourceDescription is a ceDict coming from a JobAgent, for example.
"""
resourceDict = {}
if isinstance( resourceDescription, basestring ):
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
raise ValueError( 'Illegal Resource JDL' )
self.log.verbose( classAdAgent.asJDL() )
for name in singleValueDefFields:
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in multiValueMatchFields:
if classAdAgent.lookupAttribute( name ):
if name == 'SubmitPool':
resourceDict[name] = classAdAgent.getListFromExpression( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
if classAdAgent.lookupAttribute( k ):
resourceDict[ k ] = classAdAgent.getAttributeString( k )
else:
for name in singleValueDefFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in multiValueMatchFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ):
if k in resourceDescription:
resourceDict[ k ] = resourceDescription[ k ]
return resourceDict
def __processResourceDescription( self, resourceDescription ):
# Check and form the resource description dictionary
resourceDict = {}
if type( resourceDescription ) in StringTypes:
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
return S_ERROR( 'Illegal Resource JDL' )
gLogger.verbose( classAdAgent.asJDL() )
for name in gTaskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in gTaskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute( name ):
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
if classAdAgent.lookupAttribute( 'DIRACVersion' ):
resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' )
if classAdAgent.lookupAttribute( 'VirtualOrganization' ):
resourceDict['VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization' )
else:
for name in gTaskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in gTaskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
if resourceDescription.has_key( 'DIRACVersion' ):
resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion']
if resourceDescription.has_key( 'VirtualOrganization' ):
resourceDict['VirtualOrganization'] = resourceDescription['VirtualOrganization']
return resourceDict
def __processResourceDescription( self, resourceDescription ):
# Check and form the resource description dictionary
resourceDict = {}
if type( resourceDescription ) in StringTypes:
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
return S_ERROR( 'Illegal Resource JDL' )
gLogger.verbose( classAdAgent.asJDL() )
for name in gTaskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in gTaskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute( name ):
if name == 'SubmitPool':
resourceDict[name] = classAdAgent.getListFromExpression( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
if classAdAgent.lookupAttribute( k ):
resourceDict[ k ] = classAdAgent.getAttributeString( k )
else:
for name in gTaskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in gTaskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
'PilotReference', 'PilotInfoReportedFlag', 'PilotBenchmark' ):
if k in resourceDescription:
resourceDict[ k ] = resourceDescription[ k ]
return resourceDict
def matchJob(self, resourceJDL):
""" Use Matcher service to retrieve a MPI job from Task Queue.
Returns: JobID, NumProc required, JDL and MPI flavor
Input: resourceJDL
Output: result = {JobID, JobJDL, NumProc, MPIFlavor}
"""
print "S37"
matcher = RPCClient('WorkloadManagement/Matcher', timeout = 600)
dictMatchMPI = {'Setup':'EELA-Production', 'CPUTime':6000, 'JobType':'MPI'}
result = matcher.getMatchingTaskQueues(dictMatchMPI)
if not result['OK']:
print "S38"
gLogger.info("-------------------------------------------------------------------")
gLogger.error ("Here I have to call to get normal job")
gLogger.info("-------------------------------------------------------------------")
gLogger.error (("Match not found: %s") % (result['Message']))
gLogger.info("-------------------------------------------------------------------")
return S_ERROR()
else:
if result['Value'] == {}:
gLogger.info("-------------------------------------------------------------------")
gLogger.info("Value == Empty")
gLogger.info("-------------------------------------------------------------------")
return S_ERROR()
mpiTaskQueue = result['Value']
classAdAgent = ClassAd(resourceJDL)
classAdAgent.insertAttributeString('JobType', 'MPI')
resourceJDL = str(classAdAgent.asJDL())
result = matcher.requestJob(resourceJDL)
if not result['OK']:
gLogger.error (("Request Job Error: %s") % (result['Message']))
return S_ERROR()
elif result['OK']==False:
gLogger.error (("Request Job False: %s") % (result['Message']))
return S_ERROR()
else:
gLogger.error (("Request Job OK"))
jobJDL = result['Value']['JDL']
### Review how to optimize this part (Importante)
jobID1 = ClassAd(jobJDL)
jobID = jobID1.getAttributeString('JobID')
numProc = jobID1.getAttributeString('CPUNumber')
mpiFlavor = jobID1.getAttributeString('Flavor')
result = {'JobID':jobID, 'JobJDL':jobJDL, 'NumProc': numProc,
'MPIFlavor': mpiFlavor}
print "S39"
print result
return S_OK(result)
def submitJob(self, jdl):
""" Submit one job specified by its JDL to WMS
"""
if not self.jobManagerClient:
jobManager = RPCClient(
"WorkloadManagement/JobManager", useCertificates=self.useCertificates, timeout=self.timeout
)
else:
jobManager = self.jobManagerClient
if os.path.exists(jdl):
fic = open(jdl, "r")
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is
# passed as a string
jdlString = jdl
# Check the validity of the input JDL
jdlString = jdlString.strip()
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR("Invalid job JDL")
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob)
if not result["OK"]:
return result
# Submit the job now and get the new job ID
result = jobManager.submitJob(classAdJob.asJDL())
if not result["OK"]:
return result
jobID = result["Value"]
if "requireProxyUpload" in result and result["requireProxyUpload"]:
gProxyManager.uploadProxy()
# print "Sandbox uploading"
return S_OK(jobID)
def submitJob( self, jdl, jobDescriptionObject = None ):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists( jdl ):
fic = open ( jdl, "r" )
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split('\n'):
if not line.strip().startswith( '#' ):
newJdlList.append( line )
jdlString = '\n'.join( newJdlList )
# Check the validity of the input JDL
if jdlString.find( "[" ) != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd( jdlString )
if not classAdJob.isOK():
return S_ERROR( 'Invalid job JDL' )
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox( classAdJob, jobDescriptionObject )
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
result = self.jobManager.submitJob( classAdJob.asJDL() )
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn( "Need to upload the proxy" )
return result
def submitJob(self, jdl, jobDescriptionObject=None):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists(jdl):
fic = open(jdl, "r")
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split('\n'):
if not line.strip().startswith('#'):
newJdlList.append(line)
jdlString = '\n'.join(newJdlList)
# Check the validity of the input JDL
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR('Invalid job JDL')
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject)
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
result = self.jobManager.submitJob(classAdJob.asJDL())
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn("Need to upload the proxy")
return result
def __sendJobToTaskQueue( self, job, classAdJob, siteCandidates, bannedSites ):
"""This method sends jobs to the task queue agent and if candidate sites
are defined, updates job JDL accordingly.
"""
reqJDL = classAdJob.get_expression( 'JobRequirements' )
classAddReq = ClassAd( reqJDL )
if siteCandidates:
classAddReq.insertAttributeVectorString( 'Sites', siteCandidates )
if bannedSites:
classAddReq.insertAttributeVectorString( 'BannedSites', bannedSites )
if classAdJob.lookupAttribute( "SubmitPools" ):
classAddReq.set_expression( 'SubmitPools', classAdJob.get_expression( 'SubmitPools' ) )
if classAdJob.lookupAttribute( "GridMiddleware" ):
classAddReq.set_expression( 'GridMiddleware', classAdJob.get_expression( 'GridMiddleware' ) )
if classAdJob.lookupAttribute( "PilotTypes" ):
classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotTypes' ) )
#HAck to migrate old jobs to new ones.
#DELETE ON 08/09
else:
if classAdJob.lookupAttribute( "PilotType" ):
classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotType' ) )
if classAdJob.lookupAttribute( "JobType" ):
jobTypes = [ jt for jt in classAdJob.getListFromExpression( 'JobType' ) if jt ]
classAddReq.insertAttributeVectorString( 'JobTypes', jobTypes )
#Required CE's requirements
gridCEs = [ ce for ce in classAdJob.getListFromExpression( 'GridRequiredCEs' ) if ce ]
if gridCEs:
classAddReq.insertAttributeVectorString( 'GridCEs', gridCEs )
if siteCandidates:
sites = ','.join( siteCandidates )
classAdJob.insertAttributeString( "Site", sites )
reqJDL = classAddReq.asJDL()
classAdJob.insertAttributeInt( 'JobRequirements', reqJDL )
jdl = classAdJob.asJDL()
result = self.jobDB.setJobJDL( job, jdl )
if not result['OK']:
return result
if siteCandidates:
if len( siteCandidates ) == 1:
self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, siteCandidates[0] ) )
self.jobDB.setJobAttribute( job, 'Site', siteCandidates[0] )
elif bannedSites:
remainingSites = []
for site in siteCandidates:
if not site in bannedSites:
remainingSites.append( site )
if remainingSites:
if len( remainingSites ) == 1:
self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, remainingSites[0] ) )
self.jobDB.setJobAttribute( job, 'Site', remainingSites[0] )
else:
self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( remainingSites ) ) )
result = self.jobDB.getJobAttribute(job,'Site')
siteGroup = "Multiple"
if result['OK']:
if result['Value'].startswith('Group'):
siteGroup = result['Value']
self.jobDB.setJobAttribute( job, 'Site', siteGroup )
else:
self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( siteCandidates ) ) )
result = self.jobDB.getJobAttribute(job,'Site')
siteGroup = "Multiple"
if result['OK']:
if result['Value'].startswith('Group'):
siteGroup = result['Value']
self.jobDB.setJobAttribute( job, 'Site', siteGroup )
else:
self.log.verbose( 'All sites are eligible for job %s' % job )
self.jobDB.setJobAttribute( job, 'Site', 'ANY' )
return self.setNextOptimizer( job )
def export_submitJob(self, jobDesc):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR("Can't submit using a limited proxy! (bad boy!)")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR('Failed to get job policies')
policyDict = result['Value']
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR('Job submission not authorized')
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parameteric one
jobClassAd = ClassAd(jobDesc)
parametricJob = False
if jobClassAd.lookupAttribute('Parameters'):
parametricJob = True
if jobClassAd.isAttributeList('Parameters'):
parameterList = jobClassAd.getListFromExpression('Parameters')
else:
pStep = 0
pFactor = 1
pStart = 1
nParameters = jobClassAd.getAttributeInt('Parameters')
if not nParameters:
value = jobClassAd.get_expression('Parameters')
return S_ERROR(
'Illegal value for Parameters JDL field: %s' % value)
if jobClassAd.lookupAttribute('ParameterStart'):
value = jobClassAd.get_expression(
'ParameterStart').replace('"', '')
try:
pStart = int(value)
except:
try:
pStart = float(value)
except:
return S_ERROR(
'Illegal value for ParameterStart JDL field: %s'
% value)
if jobClassAd.lookupAttribute('ParameterStep'):
pStep = jobClassAd.getAttributeInt('ParameterStep')
if not pStep:
pStep = jobClassAd.getAttributeFloat('ParameterStep')
if not pStep:
value = jobClassAd.get_expression('ParameterStep')
return S_ERROR(
'Illegal value for ParameterStep JDL field: %s'
% value)
if jobClassAd.lookupAttribute('ParameterFactor'):
pFactor = jobClassAd.getAttributeInt('ParameterFactor')
if not pFactor:
pFactor = jobClassAd.getAttributeFloat(
'ParameterFactor')
if not pFactor:
value = jobClassAd.get_expression(
'ParameterFactor')
return S_ERROR(
'Illegal value for ParameterFactor JDL field: %s'
% value)
parameterList = list()
parameterList.append(pStart)
for i in range(nParameters - 1):
parameterList.append(parameterList[i] * pFactor + pStep)
if len(parameterList) > self.maxParametricJobs:
return S_ERROR(
'The number of parametric jobs exceeded the limit of %d' %
self.maxParametricJobs)
jobDescList = []
nParam = len(parameterList) - 1
for n, p in enumerate(parameterList):
newJobDesc = jobDesc.replace('%s', str(p)).replace(
'%n',
str(n).zfill(len(str(nParam))))
newClassAd = ClassAd(newJobDesc)
for attr in ['Parameters', 'ParameterStep', 'ParameterFactor']:
newClassAd.deleteAttribute(attr)
if type(p) == type(' ') and p.startswith('{'):
newClassAd.insertAttributeInt('Parameter', str(p))
else:
newClassAd.insertAttributeString('Parameter', str(p))
newClassAd.insertAttributeInt('ParameterNumber', n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
else:
jobDescList = [jobDesc]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner,
self.ownerDN, self.ownerGroup,
self.diracSetup)
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info('Job %s added to the JobDB for %s/%s' %
(jobID, self.ownerDN, self.ownerGroup))
gJobLoggingDB.addLoggingRecord(jobID,
result['Status'],
result['MinorStatus'],
source='JobManager')
jobIDList.append(jobID)
#Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN,
self.ownerGroup)
if 'Value' not in retVal or not retVal['Value']:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result['JobID'] = result['Value']
result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
self.__sendNewJobsToMind(jobIDList)
return result
def _processResourceDescription(self, resourceDescription):
""" Check and form the resource description dictionary
resourceDescription is a ceDict coming from a JobAgent, for example.
"""
resourceDict = {}
if isinstance(resourceDescription, basestring):
classAdAgent = ClassAd(resourceDescription)
if not classAdAgent.isOK():
raise ValueError('Illegal Resource JDL')
self.log.verbose(classAdAgent.asJDL())
for name in singleValueDefFields:
if classAdAgent.lookupAttribute(name):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(
name)
for name in multiValueMatchFields:
if classAdAgent.lookupAttribute(name):
if name == 'SubmitPool':
resourceDict[
name] = classAdAgent.getListFromExpression(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(
name)
# Check if a JobID is requested
if classAdAgent.lookupAttribute('JobID'):
resourceDict['JobID'] = classAdAgent.getAttributeInt('JobID')
for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject',
'VirtualOrganization'):
if classAdAgent.lookupAttribute(k):
resourceDict[k] = classAdAgent.getAttributeString(k)
else:
for name in singleValueDefFields:
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
for name in multiValueMatchFields:
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
if 'JobID' in resourceDescription:
resourceDict['JobID'] = resourceDescription['JobID']
# Convert MaxRAM and NumberOfCores parameters into a list of tags
maxRAM = resourceDescription.get('MaxRAM')
nCores = resourceDescription.get('NumberOfProcessors')
for param, key in [(maxRAM, 'GB'), (nCores, 'Cores')]:
if param:
try:
intValue = int(param) / 1000
if intValue <= 128:
paramList = range(1, intValue + 1)
paramTags = [
'%d%s' % (par, key) for par in paramList
]
resourceDict.setdefault("Tag",
[]).extend(paramTags)
except ValueError:
pass
if 'Tag' in resourceDict:
resourceDict['Tag'] = list(set(resourceDict['Tag']))
for k in ('DIRACVersion', 'ReleaseVersion', 'ReleaseProject',
'VirtualOrganization', 'PilotReference',
'PilotBenchmark', 'PilotInfoReportedFlag'):
if k in resourceDescription:
resourceDict[k] = resourceDescription[k]
return resourceDict
def submitJob(self, jdl, jobDescriptionObject=None):
"""Submit one job specified by its JDL to WMS.
The JDL may actually be the desciption of a parametric job,
resulting in multiple DIRAC jobs submitted to the DIRAC WMS
"""
if os.path.exists(jdl):
with open(jdl, "r") as fic:
jdlString = fic.read()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
gLogger.debug("Submitting JDL", jdlString)
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split("\n"):
if not line.strip().startswith("#"):
newJdlList.append(line)
jdlString = "\n".join(newJdlList)
# Check the validity of the input JDL
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR(EWMSJDL, "Invalid job JDL")
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject)
if not result["OK"]:
return result
# Submit the job now and get the new job ID
result = getParameterVectorLength(classAdJob)
if not result["OK"]:
return result
nJobs = result["Value"]
result = self.jobManager.submitJob(classAdJob.asJDL())
if nJobs:
gLogger.debug("Applying transactional job submission")
# The server applies transactional bulk submission, we should confirm the jobs
if result["OK"]:
jobIDList = result["Value"]
if len(jobIDList) == nJobs:
# Confirm the submitted jobs
confirmed = False
for _attempt in range(3):
result = self.jobManager.confirmBulkSubmission(jobIDList)
if result["OK"]:
confirmed = True
break
time.sleep(1)
if not confirmed:
# The bulk submission failed, try to remove the created jobs
resultDelete = self.jobManager.removeJob(jobIDList)
error = "Job submission failed to confirm bulk transaction"
if not resultDelete["OK"]:
error += "; removal of created jobs failed"
return S_ERROR(EWMSSUBM, error)
else:
return S_ERROR(EWMSSUBM, "The number of submitted jobs does not match job description")
if result.get("requireProxyUpload"):
gLogger.warn("Need to upload the proxy")
return result
def _toJDL( self, xmlFile = '', jobDescriptionObject = None ): # messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
arguments = []
scriptName = 'jobDescription.xml'
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
if self.script:
if os.path.exists( self.script ):
scriptName = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptName )
else:
self.log.error( "File not found", self.script )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptName = xmlFile
self.addToInputSandbox.append( scriptName )
elif isinstance( jobDescriptionObject, StringIO.StringIO ):
self.log.verbose( "jobDescription is passed in as a StringIO object" )
else:
self.log.error( "Where's the job description?" )
arguments.append( os.path.basename( scriptName ) )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
if self.parametricWFArguments:
for name, value in self.parametricWFArguments.items():
arguments.append( "-p %s='%s'" % ( name, value ) )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle parameter sequences
if self.numberOfParameters > 0:
paramsDict, arguments = self._handleParameterSequences( paramsDict, arguments )
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.iteritems():
ptype = props['type']
value = props['value']
if isinstance( value, basestring) and re.search( ';', value ):
value = value.split( ';' )
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
if re.search( '^JDL', ptype ):
if isinstance( value, list ):
if isinstance( value[0], list ):
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorInt( name, value )
elif isinstance( value, basestring ) and value:
classadJob.insertAttributeInt( name, value )
elif isinstance( value, ( int, long, float ) ):
classadJob.insertAttributeInt( name, value )
if self.numberOfParameters > 0:
classadJob.insertAttributeInt( 'Parameters', self.numberOfParameters )
for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove( fToBeRemoved )
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def _toJDL(self, xmlFile='', jobDescriptionObject=None):
""" Creates a JDL representation of itself as a Job.
Example usage:
>>> job = Job()
>>> job._toJDL()
:param xmlFile: location of the XML file
:type xmlFile: str
:param jobDescriptionObject: if not None, it must be a StringIO object
:type jobDescriptionObject: StringIO
:returns: JDL (str)
"""
# Check if we have to do old bootstrap...
classadJob = ClassAd('[]')
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type': param.getType(), 'value': param.getValue()}
arguments = []
scriptName = 'jobDescription.xml'
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
# Messy but need to account for xml file being in /tmp/guid dir
if self.script:
if os.path.exists(self.script):
scriptName = os.path.abspath(self.script)
self.log.verbose('Found script name %s' % scriptName)
else:
self.log.warn("File not found", self.script)
else:
if xmlFile:
if os.path.exists(xmlFile):
self.log.verbose('Found XML File %s' % xmlFile)
scriptName = xmlFile
else:
self.log.warn("File not found", xmlFile)
else:
if os.path.exists('jobDescription.xml'):
scriptName = os.path.abspath('jobDescription.xml')
self.log.verbose('Found script name %s' % scriptName)
else:
self.log.warn("Job description XML file not found")
self.addToInputSandbox.append(scriptName)
elif isinstance(jobDescriptionObject, StringIO.StringIO):
self.log.verbose("jobDescription is passed in as a StringIO object")
else:
self.log.error("Where's the job description?")
arguments.append(os.path.basename(scriptName))
if 'LogLevel' in paramsDict:
if paramsDict['LogLevel']['value']:
arguments.append('-o LogLevel=%s' % (paramsDict['LogLevel']['value']))
else:
self.log.warn('Job LogLevel defined with null value')
if 'DIRACSetup' in paramsDict:
if paramsDict['DIRACSetup']['value']:
arguments.append('-o DIRAC/Setup=%s' % (paramsDict['DIRACSetup']['value']))
else:
self.log.warn('Job DIRACSetup defined with null value')
if 'JobMode' in paramsDict:
if paramsDict['JobMode']['value']:
arguments.append('-o JobMode=%s' % (paramsDict['JobMode']['value']))
else:
self.log.warn('Job Mode defined with null value')
if 'JobConfigArgs' in paramsDict:
if paramsDict['JobConfigArgs']['value']:
arguments.append('%s' % (paramsDict['JobConfigArgs']['value']))
else:
self.log.warn('JobConfigArgs defined with null value')
if self.parametricWFArguments:
for name, value in self.parametricWFArguments.iteritems():
arguments.append("-p %s='%s'" % (name, value))
classadJob.insertAttributeString('Executable', self.executable)
self.addToOutputSandbox.append(self.stderr)
self.addToOutputSandbox.append(self.stdout)
# Extract i/o sandbox parameters from steps and any input data parameters
# to do when introducing step-level api...
# To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join(self.addToInputSandbox)
if 'InputSandbox' in paramsDict:
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(';'))
paramsDict['InputSandbox']['value'] = ';'.join(uniqueInputSandbox)
self.log.verbose('Final unique Input Sandbox %s' % (';'.join(uniqueInputSandbox)))
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join(self.addToOutputSandbox)
if 'OutputSandbox' in paramsDict:
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(';'))
paramsDict['OutputSandbox']['value'] = ';'.join(uniqueOutputSandbox)
self.log.verbose('Final unique Output Sandbox %s' % (';'.join(uniqueOutputSandbox)))
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join(self.addToInputData)
if 'InputData' in paramsDict:
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements(finalInputData.split(';'))
paramsDict['InputData']['value'] = ';'.join(uniqueInputData)
self.log.verbose('Final unique Input Data %s' % (';'.join(uniqueInputData)))
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle parameter sequences
if self.numberOfParameters > 0:
paramsDict, arguments = self._handleParameterSequences(paramsDict, arguments)
classadJob.insertAttributeString('Arguments', ' '.join(arguments))
# Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.iteritems():
ptype = props['type']
value = props['value']
if isinstance(value, basestring) and re.search(';', value):
value = value.split(';')
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose('Found existing requirements: %s' % (value))
if re.search('^JDL', ptype):
if isinstance(value, list):
if isinstance(value[0], list):
classadJob.insertAttributeVectorStringList(name, value)
else:
classadJob.insertAttributeVectorInt(name, value)
elif isinstance(value, basestring) and value:
classadJob.insertAttributeInt(name, value)
elif isinstance(value, (int, long, float)):
classadJob.insertAttributeInt(name, value)
if self.numberOfParameters > 0:
classadJob.insertAttributeInt('Parameters', self.numberOfParameters)
for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove(fToBeRemoved)
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find('[')
end = jdl.rfind(']')
return jdl[(start + 1):(end - 1)]
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def _processResourceDescription( self, resourceDescription ):
""" Check and form the resource description dictionary
resourceDescription is a ceDict coming from a JobAgent, for example.
"""
resourceDict = {}
if isinstance( resourceDescription, basestring ):
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
raise ValueError( 'Illegal Resource JDL' )
self.log.verbose( classAdAgent.asJDL() )
for name in singleValueDefFields:
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in multiValueMatchFields:
if classAdAgent.lookupAttribute( name ):
if name == 'SubmitPool':
resourceDict[name] = classAdAgent.getListFromExpression( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
if classAdAgent.lookupAttribute( k ):
resourceDict[ k ] = classAdAgent.getAttributeString( k )
else:
for name in singleValueDefFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in multiValueMatchFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if 'JobID' in resourceDescription:
resourceDict['JobID'] = resourceDescription['JobID']
# Convert MaxRAM and NumberOfCores parameters into a list of tags
maxRAM = resourceDescription.get( 'MaxRAM' )
nCores = resourceDescription.get( 'NumberOfProcessors' )
for param, key in [ ( maxRAM, 'GB' ), ( nCores, 'Cores' ) ]:
if param:
try:
intValue = int( param )/1000
if intValue <= 128:
paramList = range( 1, intValue + 1 )
paramTags = [ '%d%s' % ( par, key ) for par in paramList ]
resourceDict.setdefault( "Tag", [] ).extend( paramTags )
except ValueError:
pass
if 'Tag' in resourceDict:
resourceDict['Tag'] = list( set( resourceDict['Tag'] ) )
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ):
if k in resourceDescription:
resourceDict[ k ] = resourceDescription[ k ]
return resourceDict
def _toJDL(
self, xmlFile="", jobDescriptionObject=None
): # messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
# Check if we have to do old bootstrap...
classadJob = ClassAd("[]")
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {"type": param.getType(), "value": param.getValue()}
arguments = []
scriptname = "jobDescription.xml"
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
if self.script:
if os.path.exists(self.script):
scriptname = os.path.abspath(self.script)
self.log.verbose("Found script name %s" % scriptname)
else:
self.log.error("File not found", self.script)
else:
if xmlFile:
self.log.verbose("Found XML File %s" % xmlFile)
scriptname = xmlFile
self.addToInputSandbox.append(scriptname)
elif isinstance(jobDescriptionObject, StringIO.StringIO):
self.log.verbose("jobDescription is passed in as a StringIO object")
else:
self.log.error("Where's the job description?")
arguments.append(os.path.basename(scriptname))
if paramsDict.has_key("LogLevel"):
if paramsDict["LogLevel"]["value"]:
arguments.append("-o LogLevel=%s" % (paramsDict["LogLevel"]["value"]))
else:
self.log.warn("Job LogLevel defined with null value")
if paramsDict.has_key("DIRACSetup"):
if paramsDict["DIRACSetup"]["value"]:
arguments.append("-o DIRAC/Setup=%s" % (paramsDict["DIRACSetup"]["value"]))
else:
self.log.warn("Job DIRACSetup defined with null value")
if paramsDict.has_key("JobMode"):
if paramsDict["JobMode"]["value"]:
arguments.append("-o JobMode=%s" % (paramsDict["JobMode"]["value"]))
else:
self.log.warn("Job Mode defined with null value")
if paramsDict.has_key("JobConfigArgs"):
if paramsDict["JobConfigArgs"]["value"]:
arguments.append("%s" % (paramsDict["JobConfigArgs"]["value"]))
else:
self.log.warn("JobConfigArgs defined with null value")
classadJob.insertAttributeString("Executable", self.executable)
self.addToOutputSandbox.append(self.stderr)
self.addToOutputSandbox.append(self.stdout)
# Extract i/o sandbox parameters from steps and any input data parameters
# to do when introducing step-level api...
# To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ";".join(self.addToInputSandbox)
if paramsDict.has_key("InputSandbox"):
currentFiles = paramsDict["InputSandbox"]["value"]
finalInputSandbox = currentFiles + ";" + extraFiles
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";"))
paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox)
self.log.verbose("Final unique Input Sandbox %s" % (";".join(uniqueInputSandbox)))
else:
paramsDict["InputSandbox"] = {}
paramsDict["InputSandbox"]["value"] = extraFiles
paramsDict["InputSandbox"]["type"] = "JDL"
if self.addToOutputSandbox:
extraFiles = ";".join(self.addToOutputSandbox)
if paramsDict.has_key("OutputSandbox"):
currentFiles = paramsDict["OutputSandbox"]["value"]
finalOutputSandbox = currentFiles + ";" + extraFiles
uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(";"))
paramsDict["OutputSandbox"]["value"] = ";".join(uniqueOutputSandbox)
self.log.verbose("Final unique Output Sandbox %s" % (";".join(uniqueOutputSandbox)))
else:
paramsDict["OutputSandbox"] = {}
paramsDict["OutputSandbox"]["value"] = extraFiles
paramsDict["OutputSandbox"]["type"] = "JDL"
if self.addToInputData:
extraFiles = ";".join(self.addToInputData)
if paramsDict.has_key("InputData"):
currentFiles = paramsDict["InputData"]["value"]
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ";" + extraFiles
uniqueInputData = uniqueElements(finalInputData.split(";"))
paramsDict["InputData"]["value"] = ";".join(uniqueInputData)
self.log.verbose("Final unique Input Data %s" % (";".join(uniqueInputData)))
else:
paramsDict["InputData"] = {}
paramsDict["InputData"]["value"] = extraFiles
paramsDict["InputData"]["type"] = "JDL"
# Handle here the Parametric values
if self.parametric:
for pType in ["InputData", "InputSandbox"]:
if self.parametric.has_key(pType):
if paramsDict.has_key(pType) and paramsDict[pType]["value"]:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]["value"].split(";")
tmpList = []
if type(pData[0]) == list:
for pElement in pData:
tmpList.append(currentFiles + pElement)
else:
for pElement in pData:
tmpList.append(currentFiles + [pElement])
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]["value"] = "%s"
paramsDict[pType]["type"] = "JDL"
self.parametric["files"] = self.parametric[pType]
arguments.append(" -p Parametric" + pType + "=%s")
break
if self.parametric.has_key("files"):
paramsDict["Parameters"] = {}
paramsDict["Parameters"]["value"] = self.parametric["files"]
paramsDict["Parameters"]["type"] = "JDL"
if self.parametric.has_key("GenericParameters"):
paramsDict["Parameters"] = {}
paramsDict["Parameters"]["value"] = self.parametric["GenericParameters"]
paramsDict["Parameters"]["type"] = "JDL"
arguments.append(" -p ParametricParameters=%s")
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString("Arguments", " ".join(arguments))
# Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.items():
ptype = props["type"]
value = props["value"]
if name.lower() == "requirements" and ptype == "JDL":
self.log.verbose("Found existing requirements: %s" % (value))
if re.search("^JDL", ptype):
if type(value) == list:
if type(value[0]) == list:
classadJob.insertAttributeVectorStringList(name, value)
else:
classadJob.insertAttributeVectorString(name, value)
elif value == "%s":
classadJob.insertAttributeInt(name, value)
elif not re.search(";", value) or name == "GridRequirements": # not a nice fix...
classadJob.insertAttributeString(name, value)
else:
classadJob.insertAttributeVectorString(name, value.split(";"))
for fToBeRemoved in [scriptname, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove(fToBeRemoved)
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find("[")
end = jdl.rfind("]")
return jdl[(start + 1) : (end - 1)]
def selectJob(self, resourceDescription):
""" Main job selection function to find the highest priority job
matching the resource capacity
"""
startTime = time.time()
# Check and form the resource description dictionary
resourceDict = {}
if type(resourceDescription) in StringTypes:
classAdAgent = ClassAd(resourceDescription)
if not classAdAgent.isOK():
return S_ERROR("Illegal Resource JDL")
gLogger.verbose(classAdAgent.asJDL())
for name in taskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute(name):
if name == "CPUTime":
resourceDict[name] = classAdAgent.getAttributeInt(name)
else:
resourceDict[name] = classAdAgent.getAttributeString(name)
for name in taskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute(name):
resourceDict[name] = classAdAgent.getAttributeString(name)
# Check if a JobID is requested
if classAdAgent.lookupAttribute("JobID"):
resourceDict["JobID"] = classAdAgent.getAttributeInt("JobID")
if classAdAgent.lookupAttribute("DIRACVersion"):
resourceDict["DIRACVersion"] = classAdAgent.getAttributeString("DIRACVersion")
else:
for name in taskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
for name in taskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key(name):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key("JobID"):
resourceDict["JobID"] = resourceDescription["JobID"]
if resourceDescription.has_key("DIRACVersion"):
resourceDict["DIRACVersion"] = resourceDescription["DIRACVersion"]
# Check the pilot DIRAC version
if self.checkPilotVersion:
if "DIRACVersion" in resourceDict:
if self.pilotVersion and resourceDict["DIRACVersion"] != self.pilotVersion:
return S_ERROR(
"Pilot version does not match the production version %s:%s"
% (resourceDict["DIRACVersion"], self.pilotVersion)
)
# Get common site mask and check the agent site
result = jobDB.getSiteMask(siteState="Active")
if result["OK"]:
maskList = result["Value"]
else:
return S_ERROR("Internal error: can not get site mask")
if not "Site" in resourceDict:
return S_ERROR("Missing Site Name in Resource JDL")
siteName = resourceDict["Site"]
if resourceDict["Site"] not in maskList:
if "GridCE" in resourceDict:
del resourceDict["Site"]
else:
return S_ERROR("Site not in mask and GridCE not specified")
resourceDict["Setup"] = self.serviceInfoDict["clientSetup"]
if DEBUG:
print "Resource description:"
for k, v in resourceDict.items():
print k.rjust(20), v
# Check if Job Limits are imposed onto the site
extraConditions = {}
if self.siteJobLimits:
result = self.getExtraConditions(siteName)
if result["OK"]:
extraConditions = result["Value"]
if extraConditions:
gLogger.info("Job Limits for site %s are: %s" % (siteName, str(extraConditions)))
result = taskQueueDB.matchAndGetJob(resourceDict, extraConditions=extraConditions)
if DEBUG:
print result
if not result["OK"]:
return result
result = result["Value"]
if not result["matchFound"]:
return S_ERROR("No match found")
jobID = result["jobId"]
resAtt = jobDB.getJobAttributes(jobID, ["OwnerDN", "OwnerGroup", "Status"])
if not resAtt["OK"]:
return S_ERROR("Could not retrieve job attributes")
if not resAtt["Value"]:
return S_ERROR("No attributes returned for job")
if not resAtt["Value"]["Status"] == "Waiting":
gLogger.error("Job %s matched by the TQ is not in Waiting state" % str(jobID))
result = taskQueueDB.deleteJob(jobID)
result = jobDB.setJobStatus(jobID, status="Matched", minor="Assigned")
result = jobLoggingDB.addLoggingRecord(jobID, status="Matched", minor="Assigned", source="Matcher")
result = jobDB.getJobJDL(jobID)
if not result["OK"]:
return S_ERROR("Failed to get the job JDL")
resultDict = {}
resultDict["JDL"] = result["Value"]
resultDict["JobID"] = jobID
matchTime = time.time() - startTime
gLogger.info("Match time: [%s]" % str(matchTime))
gMonitor.addMark("matchTime", matchTime)
# Get some extra stuff into the response returned
resOpt = jobDB.getJobOptParameters(jobID)
if resOpt["OK"]:
for key, value in resOpt["Value"].items():
resultDict[key] = value
resAtt = jobDB.getJobAttributes(jobID, ["OwnerDN", "OwnerGroup"])
if not resAtt["OK"]:
return S_ERROR("Could not retrieve job attributes")
if not resAtt["Value"]:
return S_ERROR("No attributes returned for job")
resultDict["DN"] = resAtt["Value"]["OwnerDN"]
resultDict["Group"] = resAtt["Value"]["OwnerGroup"]
return S_OK(resultDict)
def generateParametricJobs( jobClassAd ):
""" Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute( 'Parameters' ):
return S_OK( [ jobClassAd.asJDL() ] )
result = getParameterVectorLength( jobClassAd )
if not result['OK']:
return result
nParValues = result['Value']
if nParValues is None:
return S_ERROR(EWMSJDL, 'Can not determine the number of job parameters')
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]:
if attribute.startswith( key ):
seqID = '0' if not '.' in attribute else attribute.split( '.' )[1]
parameterDict.setdefault( seqID, {} )
if key == 'Parameters':
if jobClassAd.isAttributeList( attribute ):
parList = jobClassAd.getListFromExpression( attribute )
if len( parList ) != nParValues:
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
parameterDict[seqID]['ParameterList'] = parList
else:
if attribute != "Parameters":
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
nPar = jobClassAd.getAttributeInt( attribute )
if nPar is None:
value = jobClassAd.get_expression( attribute )
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description: %s=%s' % ( attribute, value ) )
parameterDict[seqID]['Parameters'] = nPar
else:
value = jobClassAd.getAttributeInt( attribute )
if value is None:
value = jobClassAd.getAttributeFloat( attribute )
if value is None:
value = jobClassAd.get_expression( attribute )
return S_ERROR( 'Illegal value for %s JDL field: %s' % ( attribute, value ) )
parameterDict[seqID][key] = value
if '0' in parameterDict and not parameterDict.get( '0' ):
parameterDict.pop( '0' )
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence( nParValues,
parList = parameterDict[seqID].get( 'ParameterList', [] ),
parStart = parameterDict[seqID].get( 'ParameterStart', 1 ),
parStep = parameterDict[seqID].get( 'ParameterStep', 0 ),
parFactor = parameterDict[seqID].get( 'ParameterFactor', 1 )
)
if not parList:
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len( str( nParValues - 1 ) )
for n in range( nParValues ):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace( '%n', str( n ).zfill( zLength ) )
newClassAd = ClassAd( newJobDesc )
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute( newClassAd, attribute, seqID, str( parameter ) )
for seqID in parameterLists:
for attribute in ['Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor']:
if seqID == '0':
newClassAd.deleteAttribute( attribute )
else:
newClassAd.deleteAttribute( '%s.%s' % ( attribute, seqID ) )
parameter = parameterLists[seqID][n]
if seqID == '0':
attribute = 'Parameter'
else:
attribute = 'Parameter.%s' % seqID
if isinstance( parameter, basestring) and parameter.startswith( '{' ):
newClassAd.insertAttributeInt( attribute, str( parameter ) )
else:
newClassAd.insertAttributeString( attribute, str( parameter ) )
newClassAd.insertAttributeInt( 'ParameterNumber', n )
newJDL = newClassAd.asJDL()
jobDescList.append( newJDL )
return S_OK( jobDescList )
def generateParametricJobs(jobClassAd):
"""Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute("Parameters"):
return S_OK([jobClassAd.asJDL()])
result = getParameterVectorLength(jobClassAd)
if not result["OK"]:
return result
nParValues = result["Value"]
if nParValues is None:
return S_ERROR(EWMSJDL,
"Can not determine the number of job parameters")
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [
"Parameters", "ParameterStart", "ParameterStep",
"ParameterFactor"
]:
if attribute.startswith(key):
seqID = "0" if "." not in attribute else attribute.split(
".")[1]
parameterDict.setdefault(seqID, {})
if key == "Parameters":
if jobClassAd.isAttributeList(attribute):
parList = jobClassAd.getListFromExpression(attribute)
if len(parList) != nParValues:
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description")
parameterDict[seqID]["ParameterList"] = parList
else:
if attribute != "Parameters":
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description")
nPar = jobClassAd.getAttributeInt(attribute)
if nPar is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description: %s=%s"
% (attribute, value))
parameterDict[seqID]["Parameters"] = nPar
else:
value = jobClassAd.getAttributeInt(attribute)
if value is None:
value = jobClassAd.getAttributeFloat(attribute)
if value is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
"Illegal value for %s JDL field: %s" %
(attribute, value))
parameterDict[seqID][key] = value
if "0" in parameterDict and not parameterDict.get("0"):
parameterDict.pop("0")
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence(
nParValues,
parList=parameterDict[seqID].get("ParameterList", []),
parStart=parameterDict[seqID].get("ParameterStart", 1),
parStep=parameterDict[seqID].get("ParameterStep", 0),
parFactor=parameterDict[seqID].get("ParameterFactor", 1),
)
if not parList:
return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len(str(nParValues - 1))
for n in range(nParValues):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
newClassAd = ClassAd(newJobDesc)
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute(newClassAd, attribute, seqID, str(parameter))
for seqID in parameterLists:
for attribute in [
"Parameters", "ParameterStart", "ParameterStep",
"ParameterFactor"
]:
if seqID == "0":
newClassAd.deleteAttribute(attribute)
else:
newClassAd.deleteAttribute("%s.%s" % (attribute, seqID))
parameter = parameterLists[seqID][n]
if seqID == "0":
attribute = "Parameter"
else:
attribute = "Parameter.%s" % seqID
if isinstance(parameter, str) and parameter.startswith("{"):
newClassAd.insertAttributeInt(attribute, str(parameter))
else:
newClassAd.insertAttributeString(attribute, str(parameter))
newClassAd.insertAttributeInt("ParameterNumber", n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
return S_OK(jobDescList)
def selectJob( self, resourceDescription ):
""" Main job selection function to find the highest priority job
matching the resource capacity
"""
startTime = time.time()
# Check and form the resource description dictionary
resourceDict = {}
if type( resourceDescription ) in StringTypes:
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
return S_ERROR( 'Illegal Resource JDL' )
gLogger.verbose( classAdAgent.asJDL() )
for name in gTaskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in gTaskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute( name ):
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
if classAdAgent.lookupAttribute( 'DIRACVersion' ):
resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' )
if classAdAgent.lookupAttribute( 'VirtualOrganization' ):
resourceDict['VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization' )
else:
for name in gTaskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in gTaskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
if resourceDescription.has_key( 'DIRACVersion' ):
resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion']
if resourceDescription.has_key( 'VirtualOrganization' ):
resourceDict['VirtualOrganization'] = resourceDescription['VirtualOrganization']
# Check the pilot DIRAC version
if self.checkPilotVersion:
if not 'DIRACVersion' in resourceDict:
return S_ERROR( 'Version check requested and not provided by Pilot' )
# Check if the matching Request provides a VirtualOrganization
if 'VirtualOrganization' in resourceDict:
voName = resourceDict['VirtualOrganization']
# Check if the matching Request provides an OwnerGroup
elif 'OwnerGroup' in resourceDict:
voName = getVOForGroup( resourceDict['OwnerGroup'] )
# else take the default VirtualOrganization for the installation
else:
voName = getVOForGroup( '' )
self.pilotVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( voName, self.setup ), '' )
if self.pilotVersion and resourceDict['DIRACVersion'] != self.pilotVersion:
return S_ERROR( 'Pilot version does not match the production version %s:%s' % \
( resourceDict['DIRACVersion'], self.pilotVersion ) )
# Get common site mask and check the agent site
result = gJobDB.getSiteMask( siteState = 'Active' )
if result['OK']:
maskList = result['Value']
else:
return S_ERROR( 'Internal error: can not get site mask' )
if not 'Site' in resourceDict:
return S_ERROR( 'Missing Site Name in Resource JDL' )
siteName = resourceDict['Site']
if resourceDict['Site'] not in maskList:
if 'GridCE' in resourceDict:
del resourceDict['Site']
else:
return S_ERROR( 'Site not in mask and GridCE not specified' )
resourceDict['Setup'] = self.serviceInfoDict['clientSetup']
if DEBUG:
print "Resource description:"
for key, value in resourceDict.items():
print key.rjust( 20 ), value
# Check if Job Limits are imposed onto the site
extraConditions = {}
if self.siteJobLimits:
result = self.getExtraConditions( siteName )
if result['OK']:
extraConditions = result['Value']
if extraConditions:
gLogger.info( 'Job Limits for site %s are: %s' % ( siteName, str( extraConditions ) ) )
result = gTaskQueueDB.matchAndGetJob( resourceDict, extraConditions = extraConditions )
if DEBUG:
print result
if not result['OK']:
return result
result = result['Value']
if not result['matchFound']:
return S_ERROR( 'No match found' )
jobID = result['jobId']
resAtt = gJobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] )
if not resAtt['OK']:
return S_ERROR( 'Could not retrieve job attributes' )
if not resAtt['Value']:
return S_ERROR( 'No attributes returned for job' )
if not resAtt['Value']['Status'] == 'Waiting':
gLogger.error( 'Job %s matched by the TQ is not in Waiting state' % str( jobID ) )
result = gTaskQueueDB.deleteJob( jobID )
result = gJobDB.setJobStatus( jobID, status = 'Matched', minor = 'Assigned' )
result = gJobLoggingDB.addLoggingRecord( jobID,
status = 'Matched',
minor = 'Assigned',
source = 'Matcher' )
result = gJobDB.getJobJDL( jobID )
if not result['OK']:
return S_ERROR( 'Failed to get the job JDL' )
resultDict = {}
resultDict['JDL'] = result['Value']
resultDict['JobID'] = jobID
matchTime = time.time() - startTime
gLogger.info( "Match time: [%s]" % str( matchTime ) )
gMonitor.addMark( "matchTime", matchTime )
# Get some extra stuff into the response returned
resOpt = gJobDB.getJobOptParameters( jobID )
if resOpt['OK']:
for key, value in resOpt['Value'].items():
resultDict[key] = value
resAtt = gJobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] )
if not resAtt['OK']:
return S_ERROR( 'Could not retrieve job attributes' )
if not resAtt['Value']:
return S_ERROR( 'No attributes returned for job' )
resultDict['DN'] = resAtt['Value']['OwnerDN']
resultDict['Group'] = resAtt['Value']['OwnerGroup']
return S_OK( resultDict )
def export_submitJob( self, jobDesc ):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR( "Can't submit using a limited proxy! (bad boy!)" )
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR( 'Failed to get job policies' )
policyDict = result['Value']
if not policyDict[ RIGHT_SUBMIT ]:
return S_ERROR( 'Job submission not authorized' )
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parameteric one
jobClassAd = ClassAd( jobDesc )
parametricJob = False
if jobClassAd.lookupAttribute( 'Parameters' ):
parametricJob = True
if jobClassAd.isAttributeList( 'Parameters' ):
parameterList = jobClassAd.getListFromExpression( 'Parameters' )
else:
pStep = 0
pFactor = 1
pStart = 1
nParameters = jobClassAd.getAttributeInt( 'Parameters' )
if not nParameters:
value = jobClassAd.get_expression( 'Parameters' )
return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterStart' ):
value = jobClassAd.get_expression( 'ParameterStart' ).replace( '"', '' )
try:
pStart = int( value )
except:
try:
pStart = float( value )
except:
return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterStep' ):
pStep = jobClassAd.getAttributeInt( 'ParameterStep' )
if not pStep:
pStep = jobClassAd.getAttributeFloat( 'ParameterStep' )
if not pStep:
value = jobClassAd.get_expression( 'ParameterStep' )
return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterFactor' ):
pFactor = jobClassAd.getAttributeInt( 'ParameterFactor' )
if not pFactor:
pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor' )
if not pFactor:
value = jobClassAd.get_expression( 'ParameterFactor' )
return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value )
parameterList = list()
parameterList.append( pStart )
for i in range( nParameters - 1 ):
parameterList.append( parameterList[i] * pFactor + pStep )
if len( parameterList ) > self.maxParametricJobs:
return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs )
jobDescList = []
nParam = len(parameterList) - 1
for n,p in enumerate(parameterList):
newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n).zfill(len(str(nParam))))
newClassAd = ClassAd(newJobDesc)
for attr in ['Parameters','ParameterStep','ParameterFactor']:
newClassAd.deleteAttribute(attr)
if type( p ) == type ( ' ' ) and p.startswith('{'):
newClassAd.insertAttributeInt( 'Parameter',str(p) )
else:
newClassAd.insertAttributeString( 'Parameter', str( p ) )
newClassAd.insertAttributeInt( 'ParameterNumber', n )
newJDL = newClassAd.asJDL()
jobDescList.append( newJDL )
else:
jobDescList = [ jobDesc ]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup )
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info( 'Job %s added to the JobDB for %s/%s' % ( jobID, self.ownerDN, self.ownerGroup ) )
gJobLoggingDB.addLoggingRecord( jobID, result['Status'], result['MinorStatus'], source = 'JobManager' )
jobIDList.append( jobID )
#Set persistency flag
retVal = gProxyManager.getUserPersistence( self.ownerDN, self.ownerGroup )
if 'Value' not in retVal or not retVal[ 'Value' ]:
gProxyManager.setPersistency( self.ownerDN, self.ownerGroup, True )
if parametricJob:
result = S_OK( jobIDList )
else:
result = S_OK( jobIDList[0] )
result['JobID'] = result['Value']
result[ 'requireProxyUpload' ] = self.__checkIfProxyUploadIsRequired()
self.__sendNewJobsToMind( jobIDList )
return result
def __sendJobToTaskQueue(self, job, classAdJob, siteCandidates,
bannedSites):
"""This method sends jobs to the task queue agent and if candidate sites
are defined, updates job JDL accordingly.
"""
reqJDL = classAdJob.get_expression('JobRequirements')
classAddReq = ClassAd(reqJDL)
if siteCandidates:
classAddReq.insertAttributeVectorString('Sites', siteCandidates)
if bannedSites:
classAddReq.insertAttributeVectorString('BannedSites', bannedSites)
if classAdJob.lookupAttribute("SubmitPools"):
classAddReq.set_expression(
'SubmitPools', classAdJob.get_expression('SubmitPools'))
if classAdJob.lookupAttribute("GridMiddleware"):
classAddReq.set_expression(
'GridMiddleware', classAdJob.get_expression('GridMiddleware'))
if classAdJob.lookupAttribute("PilotTypes"):
classAddReq.set_expression('PilotTypes',
classAdJob.get_expression('PilotTypes'))
#HAck to migrate old jobs to new ones.
#DELETE ON 08/09
else:
if classAdJob.lookupAttribute("PilotType"):
classAddReq.set_expression(
'PilotTypes', classAdJob.get_expression('PilotType'))
if classAdJob.lookupAttribute("JobType"):
jobTypes = [
jt for jt in classAdJob.getListFromExpression('JobType') if jt
]
classAddReq.insertAttributeVectorString('JobTypes', jobTypes)
#Required CE's requirements
gridCEs = [
ce for ce in classAdJob.getListFromExpression('GridRequiredCEs')
if ce
]
if gridCEs:
classAddReq.insertAttributeVectorString('GridCEs', gridCEs)
if siteCandidates:
sites = ','.join(siteCandidates)
classAdJob.insertAttributeString("Site", sites)
reqJDL = classAddReq.asJDL()
classAdJob.insertAttributeInt('JobRequirements', reqJDL)
jdl = classAdJob.asJDL()
result = self.jobDB.setJobJDL(job, jdl)
if not result['OK']:
return result
if siteCandidates:
if len(siteCandidates) == 1:
self.log.verbose('Individual site candidate for job %s is %s' %
(job, siteCandidates[0]))
self.jobDB.setJobAttribute(job, 'Site', siteCandidates[0])
elif bannedSites:
remainingSites = []
for site in siteCandidates:
if not site in bannedSites:
remainingSites.append(site)
if remainingSites:
if len(remainingSites) == 1:
self.log.verbose(
'Individual site candidate for job %s is %s' %
(job, remainingSites[0]))
self.jobDB.setJobAttribute(job, 'Site',
remainingSites[0])
else:
self.log.verbose('Site candidates for job %s are %s' %
(job, str(remainingSites)))
result = self.jobDB.getJobAttribute(job, 'Site')
siteGroup = "Multiple"
if result['OK']:
if result['Value'].startswith('Group'):
siteGroup = result['Value']
self.jobDB.setJobAttribute(job, 'Site', siteGroup)
else:
self.log.verbose('Site candidates for job %s are %s' %
(job, str(siteCandidates)))
result = self.jobDB.getJobAttribute(job, 'Site')
siteGroup = "Multiple"
if result['OK']:
if result['Value'].startswith('Group'):
siteGroup = result['Value']
self.jobDB.setJobAttribute(job, 'Site', siteGroup)
else:
self.log.verbose('All sites are eligible for job %s' % job)
self.jobDB.setJobAttribute(job, 'Site', 'ANY')
return self.setNextOptimizer(job)
def generateParametricJobs(jobClassAd):
""" Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute('Parameters'):
return S_OK([jobClassAd.asJDL()])
result = getParameterVectorLength(jobClassAd)
if not result['OK']:
return result
nParValues = result['Value']
if nParValues is None:
return S_ERROR(EWMSJDL,
'Can not determine the number of job parameters')
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [
'Parameters', 'ParameterStart', 'ParameterStep',
'ParameterFactor'
]:
if attribute.startswith(key):
seqID = '0' if '.' not in attribute else attribute.split(
'.')[1]
parameterDict.setdefault(seqID, {})
if key == 'Parameters':
if jobClassAd.isAttributeList(attribute):
parList = jobClassAd.getListFromExpression(attribute)
if len(parList) != nParValues:
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description')
parameterDict[seqID]['ParameterList'] = parList
else:
if attribute != "Parameters":
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description')
nPar = jobClassAd.getAttributeInt(attribute)
if nPar is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description: %s=%s'
% (attribute, value))
parameterDict[seqID]['Parameters'] = nPar
else:
value = jobClassAd.getAttributeInt(attribute)
if value is None:
value = jobClassAd.getAttributeFloat(attribute)
if value is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
'Illegal value for %s JDL field: %s' %
(attribute, value))
parameterDict[seqID][key] = value
if '0' in parameterDict and not parameterDict.get('0'):
parameterDict.pop('0')
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence(
nParValues,
parList=parameterDict[seqID].get('ParameterList', []),
parStart=parameterDict[seqID].get('ParameterStart', 1),
parStep=parameterDict[seqID].get('ParameterStep', 0),
parFactor=parameterDict[seqID].get('ParameterFactor', 1))
if not parList:
return S_ERROR(EWMSJDL, 'Inconsistent parametric job description')
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len(str(nParValues - 1))
for n in range(nParValues):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace('%n', str(n).zfill(zLength))
newClassAd = ClassAd(newJobDesc)
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute(newClassAd, attribute, seqID, str(parameter))
for seqID in parameterLists:
for attribute in [
'Parameters', 'ParameterStart', 'ParameterStep',
'ParameterFactor'
]:
if seqID == '0':
newClassAd.deleteAttribute(attribute)
else:
newClassAd.deleteAttribute('%s.%s' % (attribute, seqID))
parameter = parameterLists[seqID][n]
if seqID == '0':
attribute = 'Parameter'
else:
attribute = 'Parameter.%s' % seqID
if isinstance(parameter,
six.string_types) and parameter.startswith('{'):
newClassAd.insertAttributeInt(attribute, str(parameter))
else:
newClassAd.insertAttributeString(attribute, str(parameter))
newClassAd.insertAttributeInt('ParameterNumber', n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
return S_OK(jobDescList)
def selectJob( self, resourceDescription ):
""" Main job selection function to find the highest priority job
matching the resource capacity
"""
startTime = time.time()
# Check and form the resource description dictionary
resourceDict = {}
if type( resourceDescription ) in StringTypes:
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
return S_ERROR( 'Illegal Resource JDL' )
gLogger.verbose( classAdAgent.asJDL() )
for name in taskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in taskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute( name ):
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
if classAdAgent.lookupAttribute( 'DIRACVersion' ):
resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' )
else:
for name in taskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in taskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
if resourceDescription.has_key( 'DIRACVersion' ):
resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion']
# Check the pilot DIRAC version
if self.checkPilotVersion:
if 'DIRACVersion' in resourceDict:
if self.pilotVersion and resourceDict['DIRACVersion'] != self.pilotVersion:
return S_ERROR( 'Pilot version does not match the production version %s:%s' % \
( resourceDict['DIRACVersion'], self.pilotVersion ) )
# Get common site mask and check the agent site
result = jobDB.getSiteMask( siteState = 'Active' )
if result['OK']:
maskList = result['Value']
else:
return S_ERROR( 'Internal error: can not get site mask' )
if not 'Site' in resourceDict:
return S_ERROR( 'Missing Site Name in Resource JDL' )
siteName = resourceDict['Site']
if resourceDict['Site'] not in maskList:
if 'GridCE' in resourceDict:
del resourceDict['Site']
else:
return S_ERROR( 'Site not in mask and GridCE not specified' )
resourceDict['Setup'] = self.serviceInfoDict['clientSetup']
if DEBUG:
print "Resource description:"
for k, v in resourceDict.items():
print k.rjust( 20 ), v
# Check if Job Limits are imposed onto the site
extraConditions = {}
if self.siteJobLimits:
result = self.getExtraConditions( siteName )
if result['OK']:
extraConditions = result['Value']
if extraConditions:
gLogger.info( 'Job Limits for site %s are: %s' % ( siteName, str( extraConditions ) ) )
result = taskQueueDB.matchAndGetJob( resourceDict, extraConditions = extraConditions )
if DEBUG:
print result
if not result['OK']:
return result
result = result['Value']
if not result['matchFound']:
return S_ERROR( 'No match found' )
jobID = result['jobId']
resAtt = jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] )
if not resAtt['OK']:
return S_ERROR( 'Could not retrieve job attributes' )
if not resAtt['Value']:
return S_ERROR( 'No attributes returned for job' )
if not resAtt['Value']['Status'] == 'Waiting':
gLogger.error( 'Job %s matched by the TQ is not in Waiting state' % str( jobID ) )
result = taskQueueDB.deleteJob( jobID )
result = jobDB.setJobStatus( jobID, status = 'Matched', minor = 'Assigned' )
result = jobLoggingDB.addLoggingRecord( jobID,
status = 'Matched',
minor = 'Assigned',
source = 'Matcher' )
result = jobDB.getJobJDL( jobID )
if not result['OK']:
return S_ERROR( 'Failed to get the job JDL' )
resultDict = {}
resultDict['JDL'] = result['Value']
resultDict['JobID'] = jobID
matchTime = time.time() - startTime
gLogger.info( "Match time: [%s]" % str( matchTime ) )
gMonitor.addMark( "matchTime", matchTime )
# Get some extra stuff into the response returned
resOpt = jobDB.getJobOptParameters( jobID )
if resOpt['OK']:
for key, value in resOpt['Value'].items():
resultDict[key] = value
resAtt = jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] )
if not resAtt['OK']:
return S_ERROR( 'Could not retrieve job attributes' )
if not resAtt['Value']:
return S_ERROR( 'No attributes returned for job' )
resultDict['DN'] = resAtt['Value']['OwnerDN']
resultDict['Group'] = resAtt['Value']['OwnerGroup']
return S_OK( resultDict )
