def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def __sendJobToTaskQueue( self, job, classAdJob, siteCandidates, bannedSites ):
"""This method sends jobs to the task queue agent and if candidate sites
are defined, updates job JDL accordingly.
"""
reqJDL = classAdJob.get_expression( 'JobRequirements' )
classAddReq = ClassAd( reqJDL )
if siteCandidates:
classAddReq.insertAttributeVectorString( 'Sites', siteCandidates )
if bannedSites:
classAddReq.insertAttributeVectorString( 'BannedSites', bannedSites )
if classAdJob.lookupAttribute( "SubmitPools" ):
classAddReq.set_expression( 'SubmitPools', classAdJob.get_expression( 'SubmitPools' ) )
if classAdJob.lookupAttribute( "GridMiddleware" ):
classAddReq.set_expression( 'GridMiddleware', classAdJob.get_expression( 'GridMiddleware' ) )
if classAdJob.lookupAttribute( "PilotTypes" ):
classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotTypes' ) )
#HAck to migrate old jobs to new ones.
#DELETE ON 08/09
else:
if classAdJob.lookupAttribute( "PilotType" ):
classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotType' ) )
if classAdJob.lookupAttribute( "JobType" ):
jobTypes = [ jt for jt in classAdJob.getListFromExpression( 'JobType' ) if jt ]
classAddReq.insertAttributeVectorString( 'JobTypes', jobTypes )
#Required CE's requirements
gridCEs = [ ce for ce in classAdJob.getListFromExpression( 'GridRequiredCEs' ) if ce ]
if gridCEs:
classAddReq.insertAttributeVectorString( 'GridCEs', gridCEs )
if siteCandidates:
sites = ','.join( siteCandidates )
classAdJob.insertAttributeString( "Site", sites )
reqJDL = classAddReq.asJDL()
classAdJob.insertAttributeInt( 'JobRequirements', reqJDL )
jdl = classAdJob.asJDL()
result = self.jobDB.setJobJDL( job, jdl )
if not result['OK']:
return result
if siteCandidates:
if len( siteCandidates ) == 1:
self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, siteCandidates[0] ) )
self.jobDB.setJobAttribute( job, 'Site', siteCandidates[0] )
elif bannedSites:
remainingSites = []
for site in siteCandidates:
if not site in bannedSites:
remainingSites.append( site )
if remainingSites:
if len( remainingSites ) == 1:
self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, remainingSites[0] ) )
self.jobDB.setJobAttribute( job, 'Site', remainingSites[0] )
else:
self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( remainingSites ) ) )
result = self.jobDB.getJobAttribute(job,'Site')
siteGroup = "Multiple"
if result['OK']:
if result['Value'].startswith('Group'):
siteGroup = result['Value']
self.jobDB.setJobAttribute( job, 'Site', siteGroup )
else:
self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( siteCandidates ) ) )
result = self.jobDB.getJobAttribute(job,'Site')
siteGroup = "Multiple"
if result['OK']:
if result['Value'].startswith('Group'):
siteGroup = result['Value']
self.jobDB.setJobAttribute( job, 'Site', siteGroup )
else:
self.log.verbose( 'All sites are eligible for job %s' % job )
self.jobDB.setJobAttribute( job, 'Site', 'ANY' )
return self.setNextOptimizer( job )
def _toJDL(
self, xmlFile="", jobDescriptionObject=None
): # messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
# Check if we have to do old bootstrap...
classadJob = ClassAd("[]")
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {"type": param.getType(), "value": param.getValue()}
arguments = []
scriptname = "jobDescription.xml"
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
if self.script:
if os.path.exists(self.script):
scriptname = os.path.abspath(self.script)
self.log.verbose("Found script name %s" % scriptname)
else:
self.log.error("File not found", self.script)
else:
if xmlFile:
self.log.verbose("Found XML File %s" % xmlFile)
scriptname = xmlFile
self.addToInputSandbox.append(scriptname)
elif isinstance(jobDescriptionObject, StringIO.StringIO):
self.log.verbose("jobDescription is passed in as a StringIO object")
else:
self.log.error("Where's the job description?")
arguments.append(os.path.basename(scriptname))
if paramsDict.has_key("LogLevel"):
if paramsDict["LogLevel"]["value"]:
arguments.append("-o LogLevel=%s" % (paramsDict["LogLevel"]["value"]))
else:
self.log.warn("Job LogLevel defined with null value")
if paramsDict.has_key("DIRACSetup"):
if paramsDict["DIRACSetup"]["value"]:
arguments.append("-o DIRAC/Setup=%s" % (paramsDict["DIRACSetup"]["value"]))
else:
self.log.warn("Job DIRACSetup defined with null value")
if paramsDict.has_key("JobMode"):
if paramsDict["JobMode"]["value"]:
arguments.append("-o JobMode=%s" % (paramsDict["JobMode"]["value"]))
else:
self.log.warn("Job Mode defined with null value")
if paramsDict.has_key("JobConfigArgs"):
if paramsDict["JobConfigArgs"]["value"]:
arguments.append("%s" % (paramsDict["JobConfigArgs"]["value"]))
else:
self.log.warn("JobConfigArgs defined with null value")
classadJob.insertAttributeString("Executable", self.executable)
self.addToOutputSandbox.append(self.stderr)
self.addToOutputSandbox.append(self.stdout)
# Extract i/o sandbox parameters from steps and any input data parameters
# to do when introducing step-level api...
# To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ";".join(self.addToInputSandbox)
if paramsDict.has_key("InputSandbox"):
currentFiles = paramsDict["InputSandbox"]["value"]
finalInputSandbox = currentFiles + ";" + extraFiles
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";"))
paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox)
self.log.verbose("Final unique Input Sandbox %s" % (";".join(uniqueInputSandbox)))
else:
paramsDict["InputSandbox"] = {}
paramsDict["InputSandbox"]["value"] = extraFiles
paramsDict["InputSandbox"]["type"] = "JDL"
if self.addToOutputSandbox:
extraFiles = ";".join(self.addToOutputSandbox)
if paramsDict.has_key("OutputSandbox"):
currentFiles = paramsDict["OutputSandbox"]["value"]
finalOutputSandbox = currentFiles + ";" + extraFiles
uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(";"))
paramsDict["OutputSandbox"]["value"] = ";".join(uniqueOutputSandbox)
self.log.verbose("Final unique Output Sandbox %s" % (";".join(uniqueOutputSandbox)))
else:
paramsDict["OutputSandbox"] = {}
paramsDict["OutputSandbox"]["value"] = extraFiles
paramsDict["OutputSandbox"]["type"] = "JDL"
if self.addToInputData:
extraFiles = ";".join(self.addToInputData)
if paramsDict.has_key("InputData"):
currentFiles = paramsDict["InputData"]["value"]
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ";" + extraFiles
uniqueInputData = uniqueElements(finalInputData.split(";"))
paramsDict["InputData"]["value"] = ";".join(uniqueInputData)
self.log.verbose("Final unique Input Data %s" % (";".join(uniqueInputData)))
else:
paramsDict["InputData"] = {}
paramsDict["InputData"]["value"] = extraFiles
paramsDict["InputData"]["type"] = "JDL"
# Handle here the Parametric values
if self.parametric:
for pType in ["InputData", "InputSandbox"]:
if self.parametric.has_key(pType):
if paramsDict.has_key(pType) and paramsDict[pType]["value"]:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]["value"].split(";")
tmpList = []
if type(pData[0]) == list:
for pElement in pData:
tmpList.append(currentFiles + pElement)
else:
for pElement in pData:
tmpList.append(currentFiles + [pElement])
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]["value"] = "%s"
paramsDict[pType]["type"] = "JDL"
self.parametric["files"] = self.parametric[pType]
arguments.append(" -p Parametric" + pType + "=%s")
break
if self.parametric.has_key("files"):
paramsDict["Parameters"] = {}
paramsDict["Parameters"]["value"] = self.parametric["files"]
paramsDict["Parameters"]["type"] = "JDL"
if self.parametric.has_key("GenericParameters"):
paramsDict["Parameters"] = {}
paramsDict["Parameters"]["value"] = self.parametric["GenericParameters"]
paramsDict["Parameters"]["type"] = "JDL"
arguments.append(" -p ParametricParameters=%s")
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString("Arguments", " ".join(arguments))
# Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.items():
ptype = props["type"]
value = props["value"]
if name.lower() == "requirements" and ptype == "JDL":
self.log.verbose("Found existing requirements: %s" % (value))
if re.search("^JDL", ptype):
if type(value) == list:
if type(value[0]) == list:
classadJob.insertAttributeVectorStringList(name, value)
else:
classadJob.insertAttributeVectorString(name, value)
elif value == "%s":
classadJob.insertAttributeInt(name, value)
elif not re.search(";", value) or name == "GridRequirements": # not a nice fix...
classadJob.insertAttributeString(name, value)
else:
classadJob.insertAttributeVectorString(name, value.split(";"))
for fToBeRemoved in [scriptname, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove(fToBeRemoved)
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find("[")
end = jdl.rfind("]")
return jdl[(start + 1) : (end - 1)]
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def __sendJobToTaskQueue(self, job, classAdJob, siteCandidates,
bannedSites):
"""This method sends jobs to the task queue agent and if candidate sites
are defined, updates job JDL accordingly.
"""
reqJDL = classAdJob.get_expression('JobRequirements')
classAddReq = ClassAd(reqJDL)
if siteCandidates:
classAddReq.insertAttributeVectorString('Sites', siteCandidates)
if bannedSites:
classAddReq.insertAttributeVectorString('BannedSites', bannedSites)
if classAdJob.lookupAttribute("SubmitPools"):
classAddReq.set_expression(
'SubmitPools', classAdJob.get_expression('SubmitPools'))
if classAdJob.lookupAttribute("GridMiddleware"):
classAddReq.set_expression(
'GridMiddleware', classAdJob.get_expression('GridMiddleware'))
if classAdJob.lookupAttribute("PilotTypes"):
classAddReq.set_expression('PilotTypes',
classAdJob.get_expression('PilotTypes'))
#HAck to migrate old jobs to new ones.
#DELETE ON 08/09
else:
if classAdJob.lookupAttribute("PilotType"):
classAddReq.set_expression(
'PilotTypes', classAdJob.get_expression('PilotType'))
if classAdJob.lookupAttribute("JobType"):
jobTypes = [
jt for jt in classAdJob.getListFromExpression('JobType') if jt
]
classAddReq.insertAttributeVectorString('JobTypes', jobTypes)
#Required CE's requirements
gridCEs = [
ce for ce in classAdJob.getListFromExpression('GridRequiredCEs')
if ce
]
if gridCEs:
classAddReq.insertAttributeVectorString('GridCEs', gridCEs)
if siteCandidates:
sites = ','.join(siteCandidates)
classAdJob.insertAttributeString("Site", sites)
reqJDL = classAddReq.asJDL()
classAdJob.insertAttributeInt('JobRequirements', reqJDL)
jdl = classAdJob.asJDL()
result = self.jobDB.setJobJDL(job, jdl)
if not result['OK']:
return result
if siteCandidates:
if len(siteCandidates) == 1:
self.log.verbose('Individual site candidate for job %s is %s' %
(job, siteCandidates[0]))
self.jobDB.setJobAttribute(job, 'Site', siteCandidates[0])
elif bannedSites:
remainingSites = []
for site in siteCandidates:
if not site in bannedSites:
remainingSites.append(site)
if remainingSites:
if len(remainingSites) == 1:
self.log.verbose(
'Individual site candidate for job %s is %s' %
(job, remainingSites[0]))
self.jobDB.setJobAttribute(job, 'Site',
remainingSites[0])
else:
self.log.verbose('Site candidates for job %s are %s' %
(job, str(remainingSites)))
result = self.jobDB.getJobAttribute(job, 'Site')
siteGroup = "Multiple"
if result['OK']:
if result['Value'].startswith('Group'):
siteGroup = result['Value']
self.jobDB.setJobAttribute(job, 'Site', siteGroup)
else:
self.log.verbose('Site candidates for job %s are %s' %
(job, str(siteCandidates)))
result = self.jobDB.getJobAttribute(job, 'Site')
siteGroup = "Multiple"
if result['OK']:
if result['Value'].startswith('Group'):
siteGroup = result['Value']
self.jobDB.setJobAttribute(job, 'Site', siteGroup)
else:
self.log.verbose('All sites are eligible for job %s' % job)
self.jobDB.setJobAttribute(job, 'Site', 'ANY')
return self.setNextOptimizer(job)
