def _saveJobJDLRequest(self, jobID, jobJDL):
"""Save job JDL local to JobAgent."""
classAdJob = ClassAd(jobJDL)
classAdJob.insertAttributeString("LocalCE", self.ceName)
jdlFileName = jobID + ".jdl"
jdlFile = open(jdlFileName, "w")
jdl = classAdJob.asJDL()
jdlFile.write(jdl)
jdlFile.close()
def __saveJobJDLRequest( self, jobID, jobJDL ): """Save job JDL local to JobAgent. """ classAdJob = ClassAd( jobJDL ) classAdJob.insertAttributeString( 'LocalCE', self.ceName ) jdlFileName = jobID + '.jdl' jdlFile = open( jdlFileName, 'w' ) jdl = classAdJob.asJDL() jdlFile.write( jdl ) jdlFile.close()
def __saveJobJDLRequest(self, jobID, jobJDL):
"""Save job JDL local to JobAgent.
"""
classAdJob = ClassAd(jobJDL)
classAdJob.insertAttributeString("LocalCE", self.ceName)
jdlFileName = jobID + ".jdl"
jdlFile = open(jdlFileName, "w")
jdl = classAdJob.asJDL()
jdlFile.write(jdl)
jdlFile.close()
return S_OK(jdlFileName)
def __saveJobJDLRequest(self, jobID, jobJDL):
"""Save job JDL local to JobAgent.
"""
classAdJob = ClassAd(jobJDL)
classAdJob.insertAttributeString('LocalCE', self.ceName)
jdlFileName = jobID + '.jdl'
jdlFile = open(jdlFileName, 'w')
jdl = classAdJob.asJDL()
jdlFile.write(jdl)
jdlFile.close()
return S_OK(jdlFileName)
def matchJob(self, resourceJDL):
""" Use Matcher service to retrieve a MPI job from Task Queue.
Returns: JobID, NumProc required, JDL and MPI flavor
Input: resourceJDL
Output: result = {JobID, JobJDL, NumProc, MPIFlavor}
"""
print "S37"
matcher = RPCClient('WorkloadManagement/Matcher', timeout = 600)
dictMatchMPI = {'Setup':'EELA-Production', 'CPUTime':6000, 'JobType':'MPI'}
result = matcher.getMatchingTaskQueues(dictMatchMPI)
if not result['OK']:
print "S38"
gLogger.info("-------------------------------------------------------------------")
gLogger.error ("Here I have to call to get normal job")
gLogger.info("-------------------------------------------------------------------")
gLogger.error (("Match not found: %s") % (result['Message']))
gLogger.info("-------------------------------------------------------------------")
return S_ERROR()
else:
if result['Value'] == {}:
gLogger.info("-------------------------------------------------------------------")
gLogger.info("Value == Empty")
gLogger.info("-------------------------------------------------------------------")
return S_ERROR()
mpiTaskQueue = result['Value']
classAdAgent = ClassAd(resourceJDL)
classAdAgent.insertAttributeString('JobType', 'MPI')
resourceJDL = str(classAdAgent.asJDL())
result = matcher.requestJob(resourceJDL)
if not result['OK']:
gLogger.error (("Request Job Error: %s") % (result['Message']))
return S_ERROR()
elif result['OK']==False:
gLogger.error (("Request Job False: %s") % (result['Message']))
return S_ERROR()
else:
gLogger.error (("Request Job OK"))
jobJDL = result['Value']['JDL']
### Review how to optimize this part (Importante)
jobID1 = ClassAd(jobJDL)
jobID = jobID1.getAttributeString('JobID')
numProc = jobID1.getAttributeString('CPUNumber')
mpiFlavor = jobID1.getAttributeString('Flavor')
result = {'JobID':jobID, 'JobJDL':jobJDL, 'NumProc': numProc,
'MPIFlavor': mpiFlavor}
print "S39"
print result
return S_OK(result)
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def export_submitJob( self, jobDesc ):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR( "Can't submit using a limited proxy! (bad boy!)" )
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR( 'Failed to get job policies' )
policyDict = result['Value']
if not policyDict[ RIGHT_SUBMIT ]:
return S_ERROR( 'Job submission not authorized' )
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parameteric one
jobClassAd = ClassAd( jobDesc )
parametricJob = False
if jobClassAd.lookupAttribute( 'Parameters' ):
parametricJob = True
if jobClassAd.isAttributeList( 'Parameters' ):
parameterList = jobClassAd.getListFromExpression( 'Parameters' )
else:
pStep = 0
pFactor = 1
pStart = 1
nParameters = jobClassAd.getAttributeInt( 'Parameters' )
if not nParameters:
value = jobClassAd.get_expression( 'Parameters' )
return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterStart' ):
value = jobClassAd.get_expression( 'ParameterStart' ).replace( '"', '' )
try:
pStart = int( value )
except:
try:
pStart = float( value )
except:
return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterStep' ):
pStep = jobClassAd.getAttributeInt( 'ParameterStep' )
if not pStep:
pStep = jobClassAd.getAttributeFloat( 'ParameterStep' )
if not pStep:
value = jobClassAd.get_expression( 'ParameterStep' )
return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterFactor' ):
pFactor = jobClassAd.getAttributeInt( 'ParameterFactor' )
if not pFactor:
pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor' )
if not pFactor:
value = jobClassAd.get_expression( 'ParameterFactor' )
return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value )
parameterList = list()
parameterList.append( pStart )
for i in range( nParameters - 1 ):
parameterList.append( parameterList[i] * pFactor + pStep )
if len( parameterList ) > self.maxParametricJobs:
return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs )
jobDescList = []
nParam = len(parameterList) - 1
for n,p in enumerate(parameterList):
newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n).zfill(len(str(nParam))))
newClassAd = ClassAd(newJobDesc)
for attr in ['Parameters','ParameterStep','ParameterFactor']:
newClassAd.deleteAttribute(attr)
if type( p ) == type ( ' ' ) and p.startswith('{'):
newClassAd.insertAttributeInt( 'Parameter',str(p) )
else:
newClassAd.insertAttributeString( 'Parameter', str( p ) )
newClassAd.insertAttributeInt( 'ParameterNumber', n )
newJDL = newClassAd.asJDL()
jobDescList.append( newJDL )
else:
jobDescList = [ jobDesc ]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup )
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info( 'Job %s added to the JobDB for %s/%s' % ( jobID, self.ownerDN, self.ownerGroup ) )
gJobLoggingDB.addLoggingRecord( jobID, result['Status'], result['MinorStatus'], source = 'JobManager' )
jobIDList.append( jobID )
#Set persistency flag
retVal = gProxyManager.getUserPersistence( self.ownerDN, self.ownerGroup )
if 'Value' not in retVal or not retVal[ 'Value' ]:
gProxyManager.setPersistency( self.ownerDN, self.ownerGroup, True )
if parametricJob:
result = S_OK( jobIDList )
else:
result = S_OK( jobIDList[0] )
result['JobID'] = result['Value']
result[ 'requireProxyUpload' ] = self.__checkIfProxyUploadIsRequired()
self.__sendNewJobsToMind( jobIDList )
return result
def export_submitJob(self, jobDesc):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR("Can't submit using a limited proxy! (bad boy!)")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR('Failed to get job policies')
policyDict = result['Value']
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR('Job submission not authorized')
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parameteric one
jobClassAd = ClassAd(jobDesc)
parametricJob = False
if jobClassAd.lookupAttribute('Parameters'):
parametricJob = True
if jobClassAd.isAttributeList('Parameters'):
parameterList = jobClassAd.getListFromExpression('Parameters')
else:
pStep = 0
pFactor = 1
pStart = 1
nParameters = jobClassAd.getAttributeInt('Parameters')
if not nParameters:
value = jobClassAd.get_expression('Parameters')
return S_ERROR(
'Illegal value for Parameters JDL field: %s' % value)
if jobClassAd.lookupAttribute('ParameterStart'):
value = jobClassAd.get_expression(
'ParameterStart').replace('"', '')
try:
pStart = int(value)
except:
try:
pStart = float(value)
except:
return S_ERROR(
'Illegal value for ParameterStart JDL field: %s'
% value)
if jobClassAd.lookupAttribute('ParameterStep'):
pStep = jobClassAd.getAttributeInt('ParameterStep')
if not pStep:
pStep = jobClassAd.getAttributeFloat('ParameterStep')
if not pStep:
value = jobClassAd.get_expression('ParameterStep')
return S_ERROR(
'Illegal value for ParameterStep JDL field: %s'
% value)
if jobClassAd.lookupAttribute('ParameterFactor'):
pFactor = jobClassAd.getAttributeInt('ParameterFactor')
if not pFactor:
pFactor = jobClassAd.getAttributeFloat(
'ParameterFactor')
if not pFactor:
value = jobClassAd.get_expression(
'ParameterFactor')
return S_ERROR(
'Illegal value for ParameterFactor JDL field: %s'
% value)
parameterList = list()
parameterList.append(pStart)
for i in range(nParameters - 1):
parameterList.append(parameterList[i] * pFactor + pStep)
if len(parameterList) > self.maxParametricJobs:
return S_ERROR(
'The number of parametric jobs exceeded the limit of %d' %
self.maxParametricJobs)
jobDescList = []
nParam = len(parameterList) - 1
for n, p in enumerate(parameterList):
newJobDesc = jobDesc.replace('%s', str(p)).replace(
'%n',
str(n).zfill(len(str(nParam))))
newClassAd = ClassAd(newJobDesc)
for attr in ['Parameters', 'ParameterStep', 'ParameterFactor']:
newClassAd.deleteAttribute(attr)
if type(p) == type(' ') and p.startswith('{'):
newClassAd.insertAttributeInt('Parameter', str(p))
else:
newClassAd.insertAttributeString('Parameter', str(p))
newClassAd.insertAttributeInt('ParameterNumber', n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
else:
jobDescList = [jobDesc]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner,
self.ownerDN, self.ownerGroup,
self.diracSetup)
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info('Job %s added to the JobDB for %s/%s' %
(jobID, self.ownerDN, self.ownerGroup))
gJobLoggingDB.addLoggingRecord(jobID,
result['Status'],
result['MinorStatus'],
source='JobManager')
jobIDList.append(jobID)
#Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN,
self.ownerGroup)
if 'Value' not in retVal or not retVal['Value']:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result['JobID'] = result['Value']
result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
self.__sendNewJobsToMind(jobIDList)
return result
def _toJDL( self, xmlFile = '', jobDescriptionObject = None ): # messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
arguments = []
scriptName = 'jobDescription.xml'
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
if self.script:
if os.path.exists( self.script ):
scriptName = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptName )
else:
self.log.error( "File not found", self.script )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptName = xmlFile
self.addToInputSandbox.append( scriptName )
elif isinstance( jobDescriptionObject, StringIO.StringIO ):
self.log.verbose( "jobDescription is passed in as a StringIO object" )
else:
self.log.error( "Where's the job description?" )
arguments.append( os.path.basename( scriptName ) )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
if self.parametricWFArguments:
for name, value in self.parametricWFArguments.items():
arguments.append( "-p %s='%s'" % ( name, value ) )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle parameter sequences
if self.numberOfParameters > 0:
paramsDict, arguments = self._handleParameterSequences( paramsDict, arguments )
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.iteritems():
ptype = props['type']
value = props['value']
if isinstance( value, basestring) and re.search( ';', value ):
value = value.split( ';' )
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
if re.search( '^JDL', ptype ):
if isinstance( value, list ):
if isinstance( value[0], list ):
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorInt( name, value )
elif isinstance( value, basestring ) and value:
classadJob.insertAttributeInt( name, value )
elif isinstance( value, ( int, long, float ) ):
classadJob.insertAttributeInt( name, value )
if self.numberOfParameters > 0:
classadJob.insertAttributeInt( 'Parameters', self.numberOfParameters )
for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove( fToBeRemoved )
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def generateParametricJobs(jobClassAd):
""" Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute('Parameters'):
return S_OK([jobClassAd.asJDL()])
result = getParameterVectorLength(jobClassAd)
if not result['OK']:
return result
nParValues = result['Value']
if nParValues is None:
return S_ERROR(EWMSJDL,
'Can not determine the number of job parameters')
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [
'Parameters', 'ParameterStart', 'ParameterStep',
'ParameterFactor'
]:
if attribute.startswith(key):
seqID = '0' if '.' not in attribute else attribute.split(
'.')[1]
parameterDict.setdefault(seqID, {})
if key == 'Parameters':
if jobClassAd.isAttributeList(attribute):
parList = jobClassAd.getListFromExpression(attribute)
if len(parList) != nParValues:
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description')
parameterDict[seqID]['ParameterList'] = parList
else:
if attribute != "Parameters":
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description')
nPar = jobClassAd.getAttributeInt(attribute)
if nPar is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description: %s=%s'
% (attribute, value))
parameterDict[seqID]['Parameters'] = nPar
else:
value = jobClassAd.getAttributeInt(attribute)
if value is None:
value = jobClassAd.getAttributeFloat(attribute)
if value is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
'Illegal value for %s JDL field: %s' %
(attribute, value))
parameterDict[seqID][key] = value
if '0' in parameterDict and not parameterDict.get('0'):
parameterDict.pop('0')
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence(
nParValues,
parList=parameterDict[seqID].get('ParameterList', []),
parStart=parameterDict[seqID].get('ParameterStart', 1),
parStep=parameterDict[seqID].get('ParameterStep', 0),
parFactor=parameterDict[seqID].get('ParameterFactor', 1))
if not parList:
return S_ERROR(EWMSJDL, 'Inconsistent parametric job description')
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len(str(nParValues - 1))
for n in range(nParValues):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace('%n', str(n).zfill(zLength))
newClassAd = ClassAd(newJobDesc)
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute(newClassAd, attribute, seqID, str(parameter))
for seqID in parameterLists:
for attribute in [
'Parameters', 'ParameterStart', 'ParameterStep',
'ParameterFactor'
]:
if seqID == '0':
newClassAd.deleteAttribute(attribute)
else:
newClassAd.deleteAttribute('%s.%s' % (attribute, seqID))
parameter = parameterLists[seqID][n]
if seqID == '0':
attribute = 'Parameter'
else:
attribute = 'Parameter.%s' % seqID
if isinstance(parameter,
six.string_types) and parameter.startswith('{'):
newClassAd.insertAttributeInt(attribute, str(parameter))
else:
newClassAd.insertAttributeString(attribute, str(parameter))
newClassAd.insertAttributeInt('ParameterNumber', n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
return S_OK(jobDescList)
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def _toJDL(self, xmlFile='', jobDescriptionObject=None):
""" Creates a JDL representation of itself as a Job.
Example usage:
>>> job = Job()
>>> job._toJDL()
:param xmlFile: location of the XML file
:type xmlFile: str
:param jobDescriptionObject: if not None, it must be a StringIO object
:type jobDescriptionObject: StringIO
:returns: JDL (str)
"""
# Check if we have to do old bootstrap...
classadJob = ClassAd('[]')
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type': param.getType(), 'value': param.getValue()}
arguments = []
scriptName = 'jobDescription.xml'
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
# Messy but need to account for xml file being in /tmp/guid dir
if self.script:
if os.path.exists(self.script):
scriptName = os.path.abspath(self.script)
self.log.verbose('Found script name %s' % scriptName)
else:
self.log.warn("File not found", self.script)
else:
if xmlFile:
if os.path.exists(xmlFile):
self.log.verbose('Found XML File %s' % xmlFile)
scriptName = xmlFile
else:
self.log.warn("File not found", xmlFile)
else:
if os.path.exists('jobDescription.xml'):
scriptName = os.path.abspath('jobDescription.xml')
self.log.verbose('Found script name %s' % scriptName)
else:
self.log.warn("Job description XML file not found")
self.addToInputSandbox.append(scriptName)
elif isinstance(jobDescriptionObject, StringIO.StringIO):
self.log.verbose("jobDescription is passed in as a StringIO object")
else:
self.log.error("Where's the job description?")
arguments.append(os.path.basename(scriptName))
if 'LogLevel' in paramsDict:
if paramsDict['LogLevel']['value']:
arguments.append('-o LogLevel=%s' % (paramsDict['LogLevel']['value']))
else:
self.log.warn('Job LogLevel defined with null value')
if 'DIRACSetup' in paramsDict:
if paramsDict['DIRACSetup']['value']:
arguments.append('-o DIRAC/Setup=%s' % (paramsDict['DIRACSetup']['value']))
else:
self.log.warn('Job DIRACSetup defined with null value')
if 'JobMode' in paramsDict:
if paramsDict['JobMode']['value']:
arguments.append('-o JobMode=%s' % (paramsDict['JobMode']['value']))
else:
self.log.warn('Job Mode defined with null value')
if 'JobConfigArgs' in paramsDict:
if paramsDict['JobConfigArgs']['value']:
arguments.append('%s' % (paramsDict['JobConfigArgs']['value']))
else:
self.log.warn('JobConfigArgs defined with null value')
if self.parametricWFArguments:
for name, value in self.parametricWFArguments.iteritems():
arguments.append("-p %s='%s'" % (name, value))
classadJob.insertAttributeString('Executable', self.executable)
self.addToOutputSandbox.append(self.stderr)
self.addToOutputSandbox.append(self.stdout)
# Extract i/o sandbox parameters from steps and any input data parameters
# to do when introducing step-level api...
# To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join(self.addToInputSandbox)
if 'InputSandbox' in paramsDict:
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(';'))
paramsDict['InputSandbox']['value'] = ';'.join(uniqueInputSandbox)
self.log.verbose('Final unique Input Sandbox %s' % (';'.join(uniqueInputSandbox)))
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join(self.addToOutputSandbox)
if 'OutputSandbox' in paramsDict:
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(';'))
paramsDict['OutputSandbox']['value'] = ';'.join(uniqueOutputSandbox)
self.log.verbose('Final unique Output Sandbox %s' % (';'.join(uniqueOutputSandbox)))
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join(self.addToInputData)
if 'InputData' in paramsDict:
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements(finalInputData.split(';'))
paramsDict['InputData']['value'] = ';'.join(uniqueInputData)
self.log.verbose('Final unique Input Data %s' % (';'.join(uniqueInputData)))
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle parameter sequences
if self.numberOfParameters > 0:
paramsDict, arguments = self._handleParameterSequences(paramsDict, arguments)
classadJob.insertAttributeString('Arguments', ' '.join(arguments))
# Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.iteritems():
ptype = props['type']
value = props['value']
if isinstance(value, basestring) and re.search(';', value):
value = value.split(';')
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose('Found existing requirements: %s' % (value))
if re.search('^JDL', ptype):
if isinstance(value, list):
if isinstance(value[0], list):
classadJob.insertAttributeVectorStringList(name, value)
else:
classadJob.insertAttributeVectorInt(name, value)
elif isinstance(value, basestring) and value:
classadJob.insertAttributeInt(name, value)
elif isinstance(value, (int, long, float)):
classadJob.insertAttributeInt(name, value)
if self.numberOfParameters > 0:
classadJob.insertAttributeInt('Parameters', self.numberOfParameters)
for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove(fToBeRemoved)
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find('[')
end = jdl.rfind(']')
return jdl[(start + 1):(end - 1)]
def generateParametricJobs( jobClassAd ):
""" Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute( 'Parameters' ):
return S_OK( [ jobClassAd.asJDL() ] )
result = getParameterVectorLength( jobClassAd )
if not result['OK']:
return result
nParValues = result['Value']
if nParValues is None:
return S_ERROR(EWMSJDL, 'Can not determine the number of job parameters')
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]:
if attribute.startswith( key ):
seqID = '0' if not '.' in attribute else attribute.split( '.' )[1]
parameterDict.setdefault( seqID, {} )
if key == 'Parameters':
if jobClassAd.isAttributeList( attribute ):
parList = jobClassAd.getListFromExpression( attribute )
if len( parList ) != nParValues:
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
parameterDict[seqID]['ParameterList'] = parList
else:
if attribute != "Parameters":
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
nPar = jobClassAd.getAttributeInt( attribute )
if nPar is None:
value = jobClassAd.get_expression( attribute )
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description: %s=%s' % ( attribute, value ) )
parameterDict[seqID]['Parameters'] = nPar
else:
value = jobClassAd.getAttributeInt( attribute )
if value is None:
value = jobClassAd.getAttributeFloat( attribute )
if value is None:
value = jobClassAd.get_expression( attribute )
return S_ERROR( 'Illegal value for %s JDL field: %s' % ( attribute, value ) )
parameterDict[seqID][key] = value
if '0' in parameterDict and not parameterDict.get( '0' ):
parameterDict.pop( '0' )
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence( nParValues,
parList = parameterDict[seqID].get( 'ParameterList', [] ),
parStart = parameterDict[seqID].get( 'ParameterStart', 1 ),
parStep = parameterDict[seqID].get( 'ParameterStep', 0 ),
parFactor = parameterDict[seqID].get( 'ParameterFactor', 1 )
)
if not parList:
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len( str( nParValues - 1 ) )
for n in range( nParValues ):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace( '%n', str( n ).zfill( zLength ) )
newClassAd = ClassAd( newJobDesc )
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute( newClassAd, attribute, seqID, str( parameter ) )
for seqID in parameterLists:
for attribute in ['Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor']:
if seqID == '0':
newClassAd.deleteAttribute( attribute )
else:
newClassAd.deleteAttribute( '%s.%s' % ( attribute, seqID ) )
parameter = parameterLists[seqID][n]
if seqID == '0':
attribute = 'Parameter'
else:
attribute = 'Parameter.%s' % seqID
if isinstance( parameter, basestring) and parameter.startswith( '{' ):
newClassAd.insertAttributeInt( attribute, str( parameter ) )
else:
newClassAd.insertAttributeString( attribute, str( parameter ) )
newClassAd.insertAttributeInt( 'ParameterNumber', n )
newJDL = newClassAd.asJDL()
jobDescList.append( newJDL )
return S_OK( jobDescList )
def _toJDL(
self, xmlFile="", jobDescriptionObject=None
): # messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
# Check if we have to do old bootstrap...
classadJob = ClassAd("[]")
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {"type": param.getType(), "value": param.getValue()}
arguments = []
scriptname = "jobDescription.xml"
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
if self.script:
if os.path.exists(self.script):
scriptname = os.path.abspath(self.script)
self.log.verbose("Found script name %s" % scriptname)
else:
self.log.error("File not found", self.script)
else:
if xmlFile:
self.log.verbose("Found XML File %s" % xmlFile)
scriptname = xmlFile
self.addToInputSandbox.append(scriptname)
elif isinstance(jobDescriptionObject, StringIO.StringIO):
self.log.verbose("jobDescription is passed in as a StringIO object")
else:
self.log.error("Where's the job description?")
arguments.append(os.path.basename(scriptname))
if paramsDict.has_key("LogLevel"):
if paramsDict["LogLevel"]["value"]:
arguments.append("-o LogLevel=%s" % (paramsDict["LogLevel"]["value"]))
else:
self.log.warn("Job LogLevel defined with null value")
if paramsDict.has_key("DIRACSetup"):
if paramsDict["DIRACSetup"]["value"]:
arguments.append("-o DIRAC/Setup=%s" % (paramsDict["DIRACSetup"]["value"]))
else:
self.log.warn("Job DIRACSetup defined with null value")
if paramsDict.has_key("JobMode"):
if paramsDict["JobMode"]["value"]:
arguments.append("-o JobMode=%s" % (paramsDict["JobMode"]["value"]))
else:
self.log.warn("Job Mode defined with null value")
if paramsDict.has_key("JobConfigArgs"):
if paramsDict["JobConfigArgs"]["value"]:
arguments.append("%s" % (paramsDict["JobConfigArgs"]["value"]))
else:
self.log.warn("JobConfigArgs defined with null value")
classadJob.insertAttributeString("Executable", self.executable)
self.addToOutputSandbox.append(self.stderr)
self.addToOutputSandbox.append(self.stdout)
# Extract i/o sandbox parameters from steps and any input data parameters
# to do when introducing step-level api...
# To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ";".join(self.addToInputSandbox)
if paramsDict.has_key("InputSandbox"):
currentFiles = paramsDict["InputSandbox"]["value"]
finalInputSandbox = currentFiles + ";" + extraFiles
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";"))
paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox)
self.log.verbose("Final unique Input Sandbox %s" % (";".join(uniqueInputSandbox)))
else:
paramsDict["InputSandbox"] = {}
paramsDict["InputSandbox"]["value"] = extraFiles
paramsDict["InputSandbox"]["type"] = "JDL"
if self.addToOutputSandbox:
extraFiles = ";".join(self.addToOutputSandbox)
if paramsDict.has_key("OutputSandbox"):
currentFiles = paramsDict["OutputSandbox"]["value"]
finalOutputSandbox = currentFiles + ";" + extraFiles
uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(";"))
paramsDict["OutputSandbox"]["value"] = ";".join(uniqueOutputSandbox)
self.log.verbose("Final unique Output Sandbox %s" % (";".join(uniqueOutputSandbox)))
else:
paramsDict["OutputSandbox"] = {}
paramsDict["OutputSandbox"]["value"] = extraFiles
paramsDict["OutputSandbox"]["type"] = "JDL"
if self.addToInputData:
extraFiles = ";".join(self.addToInputData)
if paramsDict.has_key("InputData"):
currentFiles = paramsDict["InputData"]["value"]
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ";" + extraFiles
uniqueInputData = uniqueElements(finalInputData.split(";"))
paramsDict["InputData"]["value"] = ";".join(uniqueInputData)
self.log.verbose("Final unique Input Data %s" % (";".join(uniqueInputData)))
else:
paramsDict["InputData"] = {}
paramsDict["InputData"]["value"] = extraFiles
paramsDict["InputData"]["type"] = "JDL"
# Handle here the Parametric values
if self.parametric:
for pType in ["InputData", "InputSandbox"]:
if self.parametric.has_key(pType):
if paramsDict.has_key(pType) and paramsDict[pType]["value"]:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]["value"].split(";")
tmpList = []
if type(pData[0]) == list:
for pElement in pData:
tmpList.append(currentFiles + pElement)
else:
for pElement in pData:
tmpList.append(currentFiles + [pElement])
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]["value"] = "%s"
paramsDict[pType]["type"] = "JDL"
self.parametric["files"] = self.parametric[pType]
arguments.append(" -p Parametric" + pType + "=%s")
break
if self.parametric.has_key("files"):
paramsDict["Parameters"] = {}
paramsDict["Parameters"]["value"] = self.parametric["files"]
paramsDict["Parameters"]["type"] = "JDL"
if self.parametric.has_key("GenericParameters"):
paramsDict["Parameters"] = {}
paramsDict["Parameters"]["value"] = self.parametric["GenericParameters"]
paramsDict["Parameters"]["type"] = "JDL"
arguments.append(" -p ParametricParameters=%s")
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString("Arguments", " ".join(arguments))
# Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.items():
ptype = props["type"]
value = props["value"]
if name.lower() == "requirements" and ptype == "JDL":
self.log.verbose("Found existing requirements: %s" % (value))
if re.search("^JDL", ptype):
if type(value) == list:
if type(value[0]) == list:
classadJob.insertAttributeVectorStringList(name, value)
else:
classadJob.insertAttributeVectorString(name, value)
elif value == "%s":
classadJob.insertAttributeInt(name, value)
elif not re.search(";", value) or name == "GridRequirements": # not a nice fix...
classadJob.insertAttributeString(name, value)
else:
classadJob.insertAttributeVectorString(name, value.split(";"))
for fToBeRemoved in [scriptname, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove(fToBeRemoved)
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find("[")
end = jdl.rfind("]")
return jdl[(start + 1) : (end - 1)]
def generateParametricJobs(jobClassAd):
"""Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute("Parameters"):
return S_OK([jobClassAd.asJDL()])
result = getParameterVectorLength(jobClassAd)
if not result["OK"]:
return result
nParValues = result["Value"]
if nParValues is None:
return S_ERROR(EWMSJDL,
"Can not determine the number of job parameters")
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [
"Parameters", "ParameterStart", "ParameterStep",
"ParameterFactor"
]:
if attribute.startswith(key):
seqID = "0" if "." not in attribute else attribute.split(
".")[1]
parameterDict.setdefault(seqID, {})
if key == "Parameters":
if jobClassAd.isAttributeList(attribute):
parList = jobClassAd.getListFromExpression(attribute)
if len(parList) != nParValues:
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description")
parameterDict[seqID]["ParameterList"] = parList
else:
if attribute != "Parameters":
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description")
nPar = jobClassAd.getAttributeInt(attribute)
if nPar is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description: %s=%s"
% (attribute, value))
parameterDict[seqID]["Parameters"] = nPar
else:
value = jobClassAd.getAttributeInt(attribute)
if value is None:
value = jobClassAd.getAttributeFloat(attribute)
if value is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
"Illegal value for %s JDL field: %s" %
(attribute, value))
parameterDict[seqID][key] = value
if "0" in parameterDict and not parameterDict.get("0"):
parameterDict.pop("0")
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence(
nParValues,
parList=parameterDict[seqID].get("ParameterList", []),
parStart=parameterDict[seqID].get("ParameterStart", 1),
parStep=parameterDict[seqID].get("ParameterStep", 0),
parFactor=parameterDict[seqID].get("ParameterFactor", 1),
)
if not parList:
return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len(str(nParValues - 1))
for n in range(nParValues):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
newClassAd = ClassAd(newJobDesc)
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute(newClassAd, attribute, seqID, str(parameter))
for seqID in parameterLists:
for attribute in [
"Parameters", "ParameterStart", "ParameterStep",
"ParameterFactor"
]:
if seqID == "0":
newClassAd.deleteAttribute(attribute)
else:
newClassAd.deleteAttribute("%s.%s" % (attribute, seqID))
parameter = parameterLists[seqID][n]
if seqID == "0":
attribute = "Parameter"
else:
attribute = "Parameter.%s" % seqID
if isinstance(parameter, str) and parameter.startswith("{"):
newClassAd.insertAttributeInt(attribute, str(parameter))
else:
newClassAd.insertAttributeString(attribute, str(parameter))
newClassAd.insertAttributeInt("ParameterNumber", n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
return S_OK(jobDescList)
