def setExecutable(self,
executable,
arguments=None,
logFile=None,
systemConfig='ANY',
modulesNameList=None,
parametersList=None):
"""Specifies executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
:param executable: Executable
:type executable: string
:param arguments: Optional arguments to executable
:type arguments: string
:param logFile: Optional log file name
:type logFile: string
"""
kwargs = {
'executable': executable,
'arguments': arguments,
'applicationLog': logFile
}
if not isinstance(executable, str):
return self._reportError(
'Expected strings for executable and arguments', **kwargs)
if os.path.exists(executable):
self.log.verbose('Found script executable file %s' % (executable))
self.addToInputSandbox.append(executable)
logName = '%s.log' % (os.path.basename(executable))
moduleName = os.path.basename(executable)
else:
self.log.verbose('Found executable code')
logName = 'CodeOutput.log'
moduleName = 'CodeSegment'
if logFile:
if isinstance(logFile, str):
logName = str(logFile)
self.stepCount += 1
moduleName = moduleName.replace('.', '')
stepName = 'ScriptStep%s' % (self.stepCount)
if not modulesNameList:
modulesNameList = [
'CreateDataFile', 'LHCbScript', 'FileUsage',
'UserJobFinalization'
]
if not parametersList:
parametersList = [
('name', 'string', '', 'Name of executable'),
('executable', 'string', '', 'Executable Script'),
('arguments', 'string', '', 'Arguments for executable Script'),
('applicationLog', 'string', '', 'Log file name'),
('SystemConfig', 'string', '', 'binary tag')
]
step = getStepDefinition(stepName,
modulesNameList=modulesNameList,
importLine="LHCbDIRAC.Workflow.Modules",
parametersList=parametersList)
stepName = 'RunScriptStep%s' % (self.stepCount)
logPrefix = 'Script%s_' % (self.stepCount)
logName = '%s%s' % (logPrefix, logName)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue('name', moduleName)
stepInstance.setValue('applicationLog', logName)
stepInstance.setValue('executable', executable)
stepInstance.setValue('SystemConfig', systemConfig)
if arguments:
stepInstance.setValue('arguments', arguments)
return S_OK()
def setExecutable( self, executable, arguments = '', logFile = '',
modulesList = ['Script'],
parameters = [( 'executable', 'string', '', "Executable Script" ),
( 'arguments', 'string', '', 'Arguments for executable Script' ),
( 'applicationLog', 'string', '', "Log file name" )],
paramValues = [] ):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
@param executable: Executable
@type executable: string
@param arguments: Optional arguments to executable
@type arguments: string
@param logFile: Optional log file name
@type logFile: string
@param modulesList: Optional list of modules (to be used mostly when extending this method)
@type modulesList: list
@param parameters: Optional list of parameters (to be used mostly when extending this method)
@type parameters: list of tuples
@param paramValues: Optional list of parameters values (to be used mostly when extending this method)
@type parameters: list of tuples
"""
kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or \
not type( logFile ) == type( ' ' ):
return self._reportError( 'Expected strings for executable and arguments', **kwargs )
if os.path.exists( executable ):
self.log.verbose( 'Found script executable file %s' % ( executable ) )
self.addToInputSandbox.append( executable )
logName = '%s.log' % ( os.path.basename( executable ) )
else:
self.log.warn( 'The executable code could not be found locally' )
logName = 'CodeOutput.log'
if logFile:
if type( logFile ) == type( ' ' ):
logName = str(logFile)
self.stepCount += 1
stepName = 'RunScriptStep%s' % ( self.stepCount )
step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
self.addToOutputSandbox.append( logName )
stepInstance = addStepToWorkflow( self.workflow, step, stepName )
stepInstance.setValue( 'applicationLog', logName )
stepInstance.setValue( 'executable', executable )
if arguments:
stepInstance.setValue( 'arguments', arguments )
if paramValues:
for param, value in paramValues:
stepInstance.setValue( param, value )
return S_OK( stepInstance )
def setApplicationScript(self,
appName,
appVersion,
script,
arguments='',
inputData='',
inputDataType='',
poolXMLCatalog='pool_xml_catalog.xml',
logFile='',
extraPackages='',
systemConfig='ANY',
modulesNameList=None,
parametersList=None):
"""Specifies application environment and script to be executed.
For LHCb these could be e.g. Gauss, Boole, Brunel,
DaVinci etc.
The script name and any arguments should also be specified.
Input data for application script must be specified here, please note that if this is set at the job level,
e.g. via setInputData() but not above, input data is not in the scope of the specified application.
Any input data specified at the step level that is not already specified at the job level is added automatically
as a requirement for the job.
Example usage:
>>> job = LHCbJob()
>>> job.setApplicationScript('DaVinci','v19r12','myScript.py')
:param appName: Application name
:type appName: string
:param appVersion: Application version
:type appVersion: string
:param script: Script to execute
:type script: string
:param arguments: Optional arguments for script
:type arguments: string
:param inputData: Input data for application
:type inputData: single LFN or list of LFNs
:param inputDataType: Input data type for application (e.g. DATA, MDF, ETC)
:type inputDataType: string
:param arguments: Optional POOL XML Catalog name for any input data files (default is pool_xml_catalog.xml)
:type arguments: string
:param logFile: Optional log file name
:type logFile: string
:param modulesNameList: list of module names. The default is given.
:type modulesNameList: list
:param parametersList: list of parameters. The default is given.
:type parametersList: list
"""
kwargs = {
'appName': appName,
'appVersion': appVersion,
'script': script,
'arguments': arguments,
'inputData': inputData,
'inputDataType': inputDataType,
'poolXMLCatalog': poolXMLCatalog,
'logFile': logFile
}
if not isinstance(appName, str) or not isinstance(appVersion, str):
return self._reportError(
'Expected strings for application name and version', __name__,
**kwargs)
if not script or not isinstance(script, str):
return self._reportError('Expected strings for script name',
__name__, **kwargs)
if not os.path.exists(script):
return self._reportError('Script must exist locally', __name__,
**kwargs)
if logFile:
if isinstance(logFile, str):
logName = logFile
else:
return self._reportError('Expected string for log file name',
__name__, **kwargs)
else:
shortScriptName = os.path.basename(script).split('.')[0]
logName = '%s_%s_%s.log' % (appName, appVersion, shortScriptName)
self.addToInputSandbox.append(script)
if arguments:
if not isinstance(arguments, str):
return self._reportError(
'Expected string for optional script arguments', __name__,
**kwargs)
if not isinstance(poolXMLCatalog, str):
return self._reportError(
'Expected string for POOL XML Catalog name', __name__,
**kwargs)
if inputData:
if isinstance(inputData, str):
inputData = [inputData]
if not isinstance(inputData, list):
return self._reportError(
'Expected single LFN string or list of LFN(s) for inputData',
__name__, **kwargs)
if inputData != ['previousStep']:
for i in xrange(len(inputData)):
inputData[i] = inputData[i].replace('LFN:', '')
inputData = ['LFN:' + x for x in inputData]
inputDataStr = ';'.join(inputData)
self.addToInputData.append(inputDataStr)
self.stepCount += 1
stepName = '%sStep%s' % (appName, self.stepCount)
if not modulesNameList:
modulesNameList = [
'CreateDataFile', 'GaudiApplicationScript', 'FileUsage',
'UserJobFinalization'
]
if not parametersList:
parametersList = [
('applicationName', 'string', '', 'Application Name'),
('applicationVersion', 'string', '', 'Application Version'),
('applicationLog', 'string', '',
'Name of the output file of the application'),
('script', 'string', '', 'Script Name'),
('arguments', 'string', '', 'Arguments for executable'),
('poolXMLCatName', 'string', '', 'POOL XML Catalog file name'),
('inputDataType', 'string', '', 'Input Data Type'),
('inputData', 'string', '', 'Input Data'),
('extraPackages', 'string', '', 'extraPackages'),
('SystemConfig', 'string', '', 'binary tag')
]
step = getStepDefinition(stepName,
modulesNameList=modulesNameList,
importLine="LHCbDIRAC.Workflow.Modules",
parametersList=parametersList)
logPrefix = 'Step%s_' % (self.stepCount)
logName = '%s%s' % (logPrefix, logName)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue("applicationName", appName)
stepInstance.setValue("applicationVersion", appVersion)
stepInstance.setValue("script", script)
stepInstance.setValue("applicationLog", logName)
if arguments:
stepInstance.setValue("arguments", arguments)
if inputDataType:
stepInstance.setValue("inputDataType", inputDataType)
if inputData:
stepInstance.setValue("inputData", ';'.join(inputData))
if poolXMLCatalog:
stepInstance.setValue("poolXMLCatName", poolXMLCatalog)
stepInstance.setValue('extraPackages', extraPackages)
stepInstance.setValue('SystemConfig', systemConfig)
return S_OK(stepInstance)
def __configureRootModule(self, rootVersion, rootScript, rootType,
arguments, logFile, systemConfig):
""" Internal function.
Supports the root macro, python and executable wrapper functions.
"""
kwargs = {
'rootVersion': rootVersion,
'rootScript': rootScript,
'rootType': rootType,
'arguments': arguments,
'logFile': logFile
}
if not isinstance(arguments, list):
arguments = [arguments]
for param in [rootVersion, rootScript, rootType, logFile]:
if not isinstance(param, str):
return self._reportError(
'Expected strings for Root application input parameters',
__name__, **kwargs)
if not os.path.exists(rootScript):
return self._reportError(
'ROOT Script %s must exist locally' % (rootScript), __name__,
**kwargs)
self.addToInputSandbox.append(rootScript)
rootName = os.path.basename(rootScript).replace('.', '')
if logFile:
logName = logFile
else:
logName = '%s_%s.log' % (rootName, rootVersion.replace('.', ''))
self.stepCount += 1
stepName = '%sStep%s' % (rootName, self.stepCount)
modulesNameList = [
'CreateDataFile', 'RootApplication', 'FileUsage',
'UserJobFinalization'
]
parametersList = [('rootVersion', 'string', '', 'Root version'),
('rootScript', 'string', '', 'Root script'),
('rootType', 'string', '', 'Root type'),
('arguments', 'list', [],
'Optional arguments for payload'),
('applicationLog', 'string', '', 'Log file name'),
('SystemConfig', 'string', '', 'binary tag')]
step = getStepDefinition(stepName,
modulesNameList=modulesNameList,
importLine="LHCbDIRAC.Workflow.Modules",
parametersList=parametersList)
logPrefix = 'Step%s_' % (self.stepCount)
logName = '%s%s' % (logPrefix, logName)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
# Define Step and its variables
stepInstance.setValue("rootVersion", rootVersion)
stepInstance.setValue("rootType", rootType)
stepInstance.setValue("rootScript", os.path.basename(rootScript))
stepInstance.setValue("applicationLog", logName)
stepInstance.setValue("SystemConfig", systemConfig)
if arguments:
stepInstance.setValue("arguments", arguments)
return S_OK(stepInstance)
def setApplication(self,
appName,
appVersion,
optionsFiles,
inputData=None,
optionsLine=None,
inputDataType=None,
logFile=None,
events=-1,
extraPackages=None,
systemConfig='ANY',
modulesNameList=None,
parametersList=None):
"""Specifies gaudi application for DIRAC workflows, executed via gaudirun.
For LHCb these could be e.g. Gauss, Boole, Brunel, DaVinci, LHCb, etc.
The optionsFiles parameter can be the path to an options file or a list of paths to options files.
All options files are automatically appended to the job input sandBox but the first in the case of a
list is assumed to be the 'master' options file.
Input data for application script must be specified here, please note that if this is set at the job level,
e.g. via setInputData() but not above, input data is not in the scope of the specified application.
Any input data specified at the step level that is not already specified at the job level is added automatically
as a requirement for the job.
Example usage:
>>> job = LHCbJob()
>>> job.setApplication('DaVinci','v19r5',optionsFiles='MyDV.opts',
inputData=['/lhcb/production/DC06/phys-lumi2/00001501/DST/0000/00001501_00000320_5.dst'],logFile='dv.log')
:param appName: Application name
:type appName: string
:param appVersion: Application version
:type appVersion: string
:param optionsFiles: Path to options file(s) for application
:type optionsFiles: string or list
:param inputData: Input data for application (if a subset of the overall input data for a given job is required)
:type inputData: single LFN or list of LFNs
:param optionsLine: Additional options lines for application
:type optionsLine: string
:param inputDataType: Input data type for application (e.g. DATA, MDF, ETC)
:type inputDataType: string
:param logFile: Optional log file name
:type logFile: string
:param events: Optional number of events
:type events: integer
:param extraPackages: Optional extra packages
:type extraPackages: string
:param modulesNameList: list of module names. The default is given.
:type modulesNameList: list
:param parametersList: list of parameters. The default is given.
:type parametersList: list
"""
kwargs = {
'appName': appName,
'appVersion': appVersion,
'optionsFiles': optionsFiles,
'inputData': inputData,
'optionsLine': optionsLine,
'inputDataType': inputDataType,
'logFile': logFile
}
if not isinstance(appName, str) or not isinstance(appVersion, str):
return self._reportError(
'Expected strings for application name and version', __name__,
**kwargs)
if logFile:
if isinstance(logFile, str):
logName = logFile
else:
return self._reportError('Expected string for log file name',
__name__, **kwargs)
else:
logName = '%s_%s.log' % (appName, appVersion)
if not inputDataType:
inputDataType = self.inputDataType
if not isinstance(inputDataType, str):
return self._reportError('Expected string for input data type',
__name__, **kwargs)
optionsFile = None
if not optionsFiles:
return self._reportError(
'Expected string or list for optionsFiles', __name__, **kwargs)
if isinstance(optionsFiles, str):
optionsFiles = [optionsFiles]
if not isinstance(optionsFiles, list):
return self._reportError(
'Expected string or list for optionsFiles', __name__, **kwargs)
for optsFile in optionsFiles:
if not optionsFile:
self.log.verbose('Found master options file %s' % optsFile)
optionsFile = optsFile
if os.path.exists(optsFile):
self.log.verbose('Found specified options file: %s' % optsFile)
self.addToInputSandbox.append(optsFile)
optionsFile += ';%s' % optsFile
elif re.search(r'\$', optsFile):
self.log.verbose(
'Assuming %s is using an environment variable to be resolved during execution'
% optsFile)
if not optionsFile == optsFile:
optionsFile += ';%s' % optsFile
else:
return self._reportError(
'Specified options file %s does not exist' % (optsFile),
__name__, **kwargs)
# ensure optionsFile list is unique:
tmpList = optionsFile.split(';')
optionsFile = ';'.join(list(set(tmpList)))
self.log.verbose('Final options list is: %s' % optionsFile)
if inputData:
if isinstance(inputData, str):
inputData = [inputData]
if not isinstance(inputData, list):
return self._reportError(
'Expected single LFN string or list of LFN(s) for inputData',
__name__, **kwargs)
if inputData != ['previousStep']:
for i in xrange(len(inputData)):
inputData[i] = inputData[i].replace('LFN:', '')
inputData = ['LFN:' + x for x in inputData]
inputDataStr = ';'.join(inputData)
self.addToInputData.append(inputDataStr)
self.stepCount += 1
stepName = '%sStep%s' % (appName, self.stepCount)
if not modulesNameList:
modulesNameList = [
'CreateDataFile', 'GaudiApplication', 'FileUsage',
'AnalyseFileAccess', 'UserJobFinalization'
]
if not parametersList:
parametersList = [
('applicationName', 'string', '', 'Application Name'),
('applicationVersion', 'string', '', 'Application Version'),
('applicationLog', 'string', '',
'Name of the output file of the application'),
('optionsFile', 'string', '', 'Options File'),
('extraOptionsLine', 'string', '',
'This is appended to standard options'),
('inputDataType', 'string', '', 'Input Data Type'),
('inputData', 'string', '', 'Input Data'),
('numberOfEvents', 'string', '', 'Events treated'),
('extraPackages', 'string', '', 'ExtraPackages'),
('SystemConfig', 'string', '', 'binary tag')
]
step = getStepDefinition(stepName,
modulesNameList=modulesNameList,
importLine="LHCbDIRAC.Workflow.Modules",
parametersList=parametersList)
logPrefix = 'Step%s_' % (self.stepCount)
logName = '%s%s' % (logPrefix, logName)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue('applicationName', appName)
stepInstance.setValue('applicationVersion', appVersion)
stepInstance.setValue('applicationLog', logName)
if optionsFile:
stepInstance.setValue('optionsFile', optionsFile)
if optionsLine:
stepInstance.setValue('extraOptionsLine', optionsLine)
if inputDataType:
stepInstance.setValue('inputDataType', inputDataType)
if inputData:
stepInstance.setValue('inputData', ';'.join(inputData))
stepInstance.setValue('numberOfEvents', str(events))
stepInstance.setValue('extraPackages', extraPackages)
stepInstance.setValue('SystemConfig', systemConfig)
return S_OK(stepInstance)
def setExecutable( self, executable, arguments = '', logFile = '',
modulesList = None, parameters = None, paramValues = None ):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
:param executable: Executable
:type executable: string
:param arguments: Optional arguments to executable
:type arguments: string
:param logFile: Optional log file name
:type logFile: string
:param modulesList: Optional list of modules (to be used mostly when extending this method)
:type modulesList: list
:param parameters: Optional list of parameters (to be used mostly when extending this method)
:type parameters: list of tuples
:param paramValues: Optional list of parameters values (to be used mostly when extending this method)
:type parameters: list of tuples
"""
kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
if not isinstance( executable, basestring ) or not isinstance( arguments, basestring ) or \
not isinstance( logFile, basestring ):
return self._reportError( 'Expected strings for executable and arguments', **kwargs )
if os.path.exists( executable ):
self.log.verbose( 'Found script executable file %s' % ( executable ) )
self.addToInputSandbox.append( executable )
logName = '%s.log' % ( os.path.basename( executable ) )
else:
self.log.warn( 'The executable code could not be found locally' )
logName = 'CodeOutput.log'
self.stepCount += 1
stepName = 'RunScriptStep%s' % ( self.stepCount )
if logFile:
if isinstance( logFile, basestring ):
logName = str(logFile)
else:
logName = "Script%s_%s" %( self.stepCount, logName )
if not modulesList:
modulesList = ['Script']
if not parameters:
parameters = [( 'executable', 'string', '', "Executable Script" ),
( 'arguments', 'string', '', 'Arguments for executable Script' ),
( 'applicationLog', 'string', '', "Log file name" )]
step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
self.addToOutputSandbox.append( logName )
stepInstance = addStepToWorkflow( self.workflow, step, stepName )
stepInstance.setValue( 'applicationLog', logName )
stepInstance.setValue( 'executable', executable )
if arguments:
# If arguments are expressed in terms of parameters, pass them to Workflow
# These arguments will be resolved on the server side for each parametric job
if re.search( '%\(.*\)s', arguments ) or re.search( '%n', arguments ):
self.parametricWFArguments['arguments'] = arguments
else:
stepInstance.setValue( 'arguments', arguments )
if paramValues:
for param, value in paramValues:
stepInstance.setValue( param, value )
return S_OK( stepInstance )
def setExecutable( self, executable, arguments = '', logFile = '',
modulesList = ['Script'],
parameters = [( 'executable', 'string', '', "Executable Script" ),
( 'arguments', 'string', '', 'Arguments for executable Script' ),
( 'applicationLog', 'string', '', "Log file name" )],
paramValues = [] ):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
@param executable: Executable
@type executable: string
@param arguments: Optional arguments to executable
@type arguments: string
@param logFile: Optional log file name
@type logFile: string
@param modulesList: Optional list of modules (to be used mostly when extending this method)
@type modulesList: list
@param parameters: Optional list of parameters (to be used mostly when extending this method)
@type parameters: list of tuples
@param paramValues: Optional list of parameters values (to be used mostly when extending this method)
@type parameters: list of tuples
"""
kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or \
not type( logFile ) == type( ' ' ):
return self._reportError( 'Expected strings for executable and arguments', **kwargs )
if os.path.exists( executable ):
self.log.verbose( 'Found script executable file %s' % ( executable ) )
self.addToInputSandbox.append( executable )
logName = '%s.log' % ( os.path.basename( executable ) )
else:
self.log.warn( 'The executable code could not be found locally' )
logName = 'CodeOutput.log'
self.stepCount += 1
stepName = 'RunScriptStep%s' % ( self.stepCount )
if logFile:
if type( logFile ) == type( ' ' ):
logName = str(logFile)
else:
logName = "Script%s_%s" %( self.stepCount, logName )
step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
self.addToOutputSandbox.append( logName )
stepInstance = addStepToWorkflow( self.workflow, step, stepName )
stepInstance.setValue( 'applicationLog', logName )
stepInstance.setValue( 'executable', executable )
if arguments:
stepInstance.setValue( 'arguments', arguments )
if paramValues:
for param, value in paramValues:
stepInstance.setValue( param, value )
return S_OK( stepInstance )
def setExecutable(self, executable, arguments="", logFile="", modulesList=None, parameters=None, paramValues=None):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
@param executable: Executable
@type executable: string
@param arguments: Optional arguments to executable
@type arguments: string
@param logFile: Optional log file name
@type logFile: string
@param modulesList: Optional list of modules (to be used mostly when extending this method)
@type modulesList: list
@param parameters: Optional list of parameters (to be used mostly when extending this method)
@type parameters: list of tuples
@param paramValues: Optional list of parameters values (to be used mostly when extending this method)
@type parameters: list of tuples
"""
kwargs = {"executable": executable, "arguments": arguments, "logFile": logFile}
if not type(executable) == type(" ") or not type(arguments) == type(" ") or not type(logFile) == type(" "):
return self._reportError("Expected strings for executable and arguments", **kwargs)
if os.path.exists(executable):
self.log.verbose("Found script executable file %s" % (executable))
self.addToInputSandbox.append(executable)
logName = "%s.log" % (os.path.basename(executable))
else:
self.log.warn("The executable code could not be found locally")
logName = "CodeOutput.log"
self.stepCount += 1
stepName = "RunScriptStep%s" % (self.stepCount)
if logFile:
if type(logFile) == type(" "):
logName = str(logFile)
else:
logName = "Script%s_%s" % (self.stepCount, logName)
if not modulesList:
modulesList = ["Script"]
if not parameters:
parameters = [
("executable", "string", "", "Executable Script"),
("arguments", "string", "", "Arguments for executable Script"),
("applicationLog", "string", "", "Log file name"),
]
step = getStepDefinition("ScriptStep%s" % (self.stepCount), modulesList, parametersList=parameters)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue("applicationLog", logName)
stepInstance.setValue("executable", executable)
if arguments:
stepInstance.setValue("arguments", arguments)
if paramValues:
for param, value in paramValues:
stepInstance.setValue(param, value)
return S_OK(stepInstance)
def setExecutable(self, executable, arguments='', logFile='',
modulesList=None, parameters=None, paramValues=None):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
:param executable: Executable
:type executable: string
:param arguments: Optional arguments to executable
:type arguments: string
:param logFile: Optional log file name
:type logFile: string
:param modulesList: Optional list of modules (to be used mostly when extending this method)
:type modulesList: python:list
:param parameters: Optional list of parameters (to be used mostly when extending this method)
:type parameters: python:list of tuples
:param paramValues: Optional list of parameters values (to be used mostly when extending this method)
:type parameters: python:list of tuples
"""
kwargs = {'executable': executable, 'arguments': arguments, 'logFile': logFile}
if not isinstance(executable, basestring) or not isinstance(arguments, six.string_types) or \
not isinstance(logFile, six.string_types):
return self._reportError('Expected strings for executable and arguments', **kwargs)
if os.path.exists(executable):
self.log.verbose('Found script executable file %s' % (executable))
self.addToInputSandbox.append(executable)
logName = '%s.log' % (os.path.basename(executable))
else:
self.log.warn('The executable code could not be found locally')
logName = 'CodeOutput.log'
self.stepCount += 1
stepName = 'RunScriptStep%s' % (self.stepCount)
if logFile:
if isinstance(logFile, six.string_types):
logName = str(logFile)
else:
logName = "Script%s_%s" % (self.stepCount, logName)
if not modulesList:
modulesList = ['Script']
if not parameters:
parameters = [('executable', 'string', '', "Executable Script"),
('arguments', 'string', '', 'Arguments for executable Script'),
('applicationLog', 'string', '', "Log file name")]
step = getStepDefinition('ScriptStep%s' % (self.stepCount), modulesList, parametersList=parameters)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue('applicationLog', logName)
stepInstance.setValue('executable', executable)
if arguments:
# If arguments are expressed in terms of parameters, pass them to Workflow
# These arguments will be resolved on the server side for each parametric job
if re.search(r'%\(.*\)s', arguments) or re.search('%n', arguments):
self.parametricWFArguments['arguments'] = arguments
else:
stepInstance.setValue('arguments', arguments)
if paramValues:
for param, value in paramValues:
stepInstance.setValue(param, value)
return S_OK(stepInstance)