def _toJDL( self, xmlFile = '', jobDescriptionObject = None ): # messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
arguments = []
scriptName = 'jobDescription.xml'
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
if self.script:
if os.path.exists( self.script ):
scriptName = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptName )
else:
self.log.error( "File not found", self.script )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptName = xmlFile
self.addToInputSandbox.append( scriptName )
elif isinstance( jobDescriptionObject, StringIO.StringIO ):
self.log.verbose( "jobDescription is passed in as a StringIO object" )
else:
self.log.error( "Where's the job description?" )
arguments.append( os.path.basename( scriptName ) )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
if self.parametricWFArguments:
for name, value in self.parametricWFArguments.items():
arguments.append( "-p %s='%s'" % ( name, value ) )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle parameter sequences
if self.numberOfParameters > 0:
paramsDict, arguments = self._handleParameterSequences( paramsDict, arguments )
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.iteritems():
ptype = props['type']
value = props['value']
if isinstance( value, basestring) and re.search( ';', value ):
value = value.split( ';' )
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
if re.search( '^JDL', ptype ):
if isinstance( value, list ):
if isinstance( value[0], list ):
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorInt( name, value )
elif isinstance( value, basestring ) and value:
classadJob.insertAttributeInt( name, value )
elif isinstance( value, ( int, long, float ) ):
classadJob.insertAttributeInt( name, value )
if self.numberOfParameters > 0:
classadJob.insertAttributeInt( 'Parameters', self.numberOfParameters )
for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove( fToBeRemoved )
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def _toJDL(self, xmlFile='', jobDescriptionObject=None):
""" Creates a JDL representation of itself as a Job.
Example usage:
>>> job = Job()
>>> job._toJDL()
:param xmlFile: location of the XML file
:type xmlFile: str
:param jobDescriptionObject: if not None, it must be a StringIO object
:type jobDescriptionObject: StringIO
:returns: JDL (str)
"""
# Check if we have to do old bootstrap...
classadJob = ClassAd('[]')
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type': param.getType(), 'value': param.getValue()}
arguments = []
scriptName = 'jobDescription.xml'
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
# Messy but need to account for xml file being in /tmp/guid dir
if self.script:
if os.path.exists(self.script):
scriptName = os.path.abspath(self.script)
self.log.verbose('Found script name %s' % scriptName)
else:
self.log.warn("File not found", self.script)
else:
if xmlFile:
if os.path.exists(xmlFile):
self.log.verbose('Found XML File %s' % xmlFile)
scriptName = xmlFile
else:
self.log.warn("File not found", xmlFile)
else:
if os.path.exists('jobDescription.xml'):
scriptName = os.path.abspath('jobDescription.xml')
self.log.verbose('Found script name %s' % scriptName)
else:
self.log.warn("Job description XML file not found")
self.addToInputSandbox.append(scriptName)
elif isinstance(jobDescriptionObject, StringIO.StringIO):
self.log.verbose("jobDescription is passed in as a StringIO object")
else:
self.log.error("Where's the job description?")
arguments.append(os.path.basename(scriptName))
if 'LogLevel' in paramsDict:
if paramsDict['LogLevel']['value']:
arguments.append('-o LogLevel=%s' % (paramsDict['LogLevel']['value']))
else:
self.log.warn('Job LogLevel defined with null value')
if 'DIRACSetup' in paramsDict:
if paramsDict['DIRACSetup']['value']:
arguments.append('-o DIRAC/Setup=%s' % (paramsDict['DIRACSetup']['value']))
else:
self.log.warn('Job DIRACSetup defined with null value')
if 'JobMode' in paramsDict:
if paramsDict['JobMode']['value']:
arguments.append('-o JobMode=%s' % (paramsDict['JobMode']['value']))
else:
self.log.warn('Job Mode defined with null value')
if 'JobConfigArgs' in paramsDict:
if paramsDict['JobConfigArgs']['value']:
arguments.append('%s' % (paramsDict['JobConfigArgs']['value']))
else:
self.log.warn('JobConfigArgs defined with null value')
if self.parametricWFArguments:
for name, value in self.parametricWFArguments.iteritems():
arguments.append("-p %s='%s'" % (name, value))
classadJob.insertAttributeString('Executable', self.executable)
self.addToOutputSandbox.append(self.stderr)
self.addToOutputSandbox.append(self.stdout)
# Extract i/o sandbox parameters from steps and any input data parameters
# to do when introducing step-level api...
# To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join(self.addToInputSandbox)
if 'InputSandbox' in paramsDict:
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(';'))
paramsDict['InputSandbox']['value'] = ';'.join(uniqueInputSandbox)
self.log.verbose('Final unique Input Sandbox %s' % (';'.join(uniqueInputSandbox)))
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join(self.addToOutputSandbox)
if 'OutputSandbox' in paramsDict:
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(';'))
paramsDict['OutputSandbox']['value'] = ';'.join(uniqueOutputSandbox)
self.log.verbose('Final unique Output Sandbox %s' % (';'.join(uniqueOutputSandbox)))
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join(self.addToInputData)
if 'InputData' in paramsDict:
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements(finalInputData.split(';'))
paramsDict['InputData']['value'] = ';'.join(uniqueInputData)
self.log.verbose('Final unique Input Data %s' % (';'.join(uniqueInputData)))
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle parameter sequences
if self.numberOfParameters > 0:
paramsDict, arguments = self._handleParameterSequences(paramsDict, arguments)
classadJob.insertAttributeString('Arguments', ' '.join(arguments))
# Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.iteritems():
ptype = props['type']
value = props['value']
if isinstance(value, basestring) and re.search(';', value):
value = value.split(';')
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose('Found existing requirements: %s' % (value))
if re.search('^JDL', ptype):
if isinstance(value, list):
if isinstance(value[0], list):
classadJob.insertAttributeVectorStringList(name, value)
else:
classadJob.insertAttributeVectorInt(name, value)
elif isinstance(value, basestring) and value:
classadJob.insertAttributeInt(name, value)
elif isinstance(value, (int, long, float)):
classadJob.insertAttributeInt(name, value)
if self.numberOfParameters > 0:
classadJob.insertAttributeInt('Parameters', self.numberOfParameters)
for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove(fToBeRemoved)
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find('[')
end = jdl.rfind(']')
return jdl[(start + 1):(end - 1)]
