def test__getStepDefinition(self):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition("Script")
body = importLine.replace("<MODULE>", "Script")
gaudiApp.setDescription(
getattr(
__import__("%s.%s" % ("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), "__doc__"
)
)
gaudiApp.setBody(body)
genBKReport = ModuleDefinition("FailoverRequest")
body = importLine.replace("<MODULE>", "FailoverRequest")
genBKReport.setDescription(
getattr(
__import__("%s.%s" % ("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"]),
"__doc__",
)
)
genBKReport.setBody(body)
# step
appDefn = StepDefinition("App_Step")
appDefn.addModule(gaudiApp)
appDefn.createModuleInstance("Script", "Script")
appDefn.addModule(genBKReport)
appDefn.createModuleInstance("FailoverRequest", "FailoverRequest")
appDefn.addParameterLinked(gaudiApp.parameters)
stepDef = getStepDefinition("App_Step", ["Script", "FailoverRequest"])
self.assertTrue(str(appDefn) == str(stepDef))
self.job._addParameter(appDefn, "name", "type", "value", "desc")
self.job._addParameter(appDefn, "name1", "type1", "value1", "desc1")
stepDef = getStepDefinition(
"App_Step",
["Script", "FailoverRequest"],
parametersList=[["name", "type", "value", "desc"], ["name1", "type1", "value1", "desc1"]],
)
self.assertTrue(str(appDefn) == str(stepDef))
def test__getStepDefinition(self):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition('Script')
body = importLine.replace('<MODULE>', 'Script')
gaudiApp.setDescription(
getattr(
__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'Script'),
globals(), locals(), ['__doc__']), "__doc__"))
gaudiApp.setBody(body)
genBKReport = ModuleDefinition('FailoverRequest')
body = importLine.replace('<MODULE>', 'FailoverRequest')
genBKReport.setDescription(
getattr(
__import__(
"%s.%s" % ('DIRAC.Workflow.Modules', 'FailoverRequest'),
globals(), locals(), ['__doc__']), "__doc__"))
genBKReport.setBody(body)
# step
appDefn = StepDefinition('App_Step')
appDefn.addModule(gaudiApp)
appDefn.createModuleInstance('Script', 'Script')
appDefn.addModule(genBKReport)
appDefn.createModuleInstance('FailoverRequest', 'FailoverRequest')
appDefn.addParameterLinked(gaudiApp.parameters)
stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'])
self.assert_(str(appDefn) == str(stepDef))
self.job._addParameter(appDefn, 'name', 'type', 'value', 'desc')
self.job._addParameter(appDefn, 'name1', 'type1', 'value1', 'desc1')
stepDef = getStepDefinition(
'App_Step', ['Script', 'FailoverRequest'],
parametersList=[['name', 'type', 'value', 'desc'],
['name1', 'type1', 'value1', 'desc1']])
self.assert_(str(appDefn) == str(stepDef))
def test__getStepDefinition( self ):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition( 'Script' )
body = importLine.replace( '<MODULE>', 'Script' )
gaudiApp.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'Script' ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
gaudiApp.setBody( body )
genBKReport = ModuleDefinition( 'FailoverRequest' )
body = importLine.replace( '<MODULE>', 'FailoverRequest' )
genBKReport.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'FailoverRequest' ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
genBKReport.setBody( body )
# step
appDefn = StepDefinition( 'App_Step' )
appDefn.addModule( gaudiApp )
appDefn.createModuleInstance( 'Script', 'Script' )
appDefn.addModule( genBKReport )
appDefn.createModuleInstance( 'FailoverRequest', 'FailoverRequest' )
appDefn.addParameterLinked( gaudiApp.parameters )
stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'] )
self.assertTrue( str( appDefn ) == str( stepDef ) )
self.job._addParameter( appDefn, 'name', 'type', 'value', 'desc' )
self.job._addParameter( appDefn, 'name1', 'type1', 'value1', 'desc1' )
stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'],
parametersList = [[ 'name', 'type', 'value', 'desc' ],
[ 'name1', 'type1', 'value1', 'desc1' ]] )
self.assertTrue( str( appDefn ) == str( stepDef ) )
def addFinalizationStep(self, modulesList=None):
""" Add the finalization step to the workflow (some defaults are inserted)
:param list modulesList: the list of modules names this step is made of. If None, a default is taken.
:returns: None
"""
if modulesList is None:
modulesList = [
'UploadOutputData', 'UploadLogFile', 'UploadMC',
'FailoverRequest'
]
if 'Job_Finalization' not in self.LHCbJob.workflow.step_definitions.keys(
):
jobFinalizationStepDef = getStepDefinition(
'Job_Finalization',
importLine='LHCbDIRAC.Workflow.Modules',
modulesNameList=modulesList)
self.LHCbJob.workflow.addStep(jobFinalizationStepDef)
# create the step instance add it to the wf
self.LHCbJob.workflow.createStepInstance('Job_Finalization',
'finalization')
def setExecutable( self, executable, arguments = '', logFile = '',
modulesList = ['Script'],
parameters = [( 'executable', 'string', '', "Executable Script" ),
( 'arguments', 'string', '', 'Arguments for executable Script' ),
( 'applicationLog', 'string', '', "Log file name" )],
paramValues = [] ):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
@param executable: Executable
@type executable: string
@param arguments: Optional arguments to executable
@type arguments: string
@param logFile: Optional log file name
@type logFile: string
@param modulesList: Optional list of modules (to be used mostly when extending this method)
@type modulesList: list
@param parameters: Optional list of parameters (to be used mostly when extending this method)
@type parameters: list of tuples
@param paramValues: Optional list of parameters values (to be used mostly when extending this method)
@type parameters: list of tuples
"""
kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or \
not type( logFile ) == type( ' ' ):
return self._reportError( 'Expected strings for executable and arguments', **kwargs )
if os.path.exists( executable ):
self.log.verbose( 'Found script executable file %s' % ( executable ) )
self.addToInputSandbox.append( executable )
logName = '%s.log' % ( os.path.basename( executable ) )
else:
self.log.warn( 'The executable code could not be found locally' )
logName = 'CodeOutput.log'
if logFile:
if type( logFile ) == type( ' ' ):
logName = str(logFile)
self.stepCount += 1
stepName = 'RunScriptStep%s' % ( self.stepCount )
step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
self.addToOutputSandbox.append( logName )
stepInstance = addStepToWorkflow( self.workflow, step, stepName )
stepInstance.setValue( 'applicationLog', logName )
stepInstance.setValue( 'executable', executable )
if arguments:
stepInstance.setValue( 'arguments', arguments )
if paramValues:
for param, value in paramValues:
stepInstance.setValue( param, value )
return S_OK( stepInstance )
def setExecutable(self,
executable,
arguments=None,
logFile=None,
systemConfig='ANY',
modulesNameList=None,
parametersList=None):
"""Specifies executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
:param executable: Executable
:type executable: string
:param arguments: Optional arguments to executable
:type arguments: string
:param logFile: Optional log file name
:type logFile: string
"""
kwargs = {
'executable': executable,
'arguments': arguments,
'applicationLog': logFile
}
if not isinstance(executable, str):
return self._reportError(
'Expected strings for executable and arguments', **kwargs)
if os.path.exists(executable):
self.log.verbose('Found script executable file %s' % (executable))
self.addToInputSandbox.append(executable)
logName = '%s.log' % (os.path.basename(executable))
moduleName = os.path.basename(executable)
else:
self.log.verbose('Found executable code')
logName = 'CodeOutput.log'
moduleName = 'CodeSegment'
if logFile:
if isinstance(logFile, str):
logName = str(logFile)
self.stepCount += 1
moduleName = moduleName.replace('.', '')
stepName = 'ScriptStep%s' % (self.stepCount)
if not modulesNameList:
modulesNameList = [
'CreateDataFile', 'LHCbScript', 'FileUsage',
'UserJobFinalization'
]
if not parametersList:
parametersList = [
('name', 'string', '', 'Name of executable'),
('executable', 'string', '', 'Executable Script'),
('arguments', 'string', '', 'Arguments for executable Script'),
('applicationLog', 'string', '', 'Log file name'),
('SystemConfig', 'string', '', 'binary tag')
]
step = getStepDefinition(stepName,
modulesNameList=modulesNameList,
importLine="LHCbDIRAC.Workflow.Modules",
parametersList=parametersList)
stepName = 'RunScriptStep%s' % (self.stepCount)
logPrefix = 'Script%s_' % (self.stepCount)
logName = '%s%s' % (logPrefix, logName)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue('name', moduleName)
stepInstance.setValue('applicationLog', logName)
stepInstance.setValue('executable', executable)
stepInstance.setValue('SystemConfig', systemConfig)
if arguments:
stepInstance.setValue('arguments', arguments)
return S_OK()
def __configureRootModule(self, rootVersion, rootScript, rootType,
arguments, logFile, systemConfig):
""" Internal function.
Supports the root macro, python and executable wrapper functions.
"""
kwargs = {
'rootVersion': rootVersion,
'rootScript': rootScript,
'rootType': rootType,
'arguments': arguments,
'logFile': logFile
}
if not isinstance(arguments, list):
arguments = [arguments]
for param in [rootVersion, rootScript, rootType, logFile]:
if not isinstance(param, str):
return self._reportError(
'Expected strings for Root application input parameters',
__name__, **kwargs)
if not os.path.exists(rootScript):
return self._reportError(
'ROOT Script %s must exist locally' % (rootScript), __name__,
**kwargs)
self.addToInputSandbox.append(rootScript)
rootName = os.path.basename(rootScript).replace('.', '')
if logFile:
logName = logFile
else:
logName = '%s_%s.log' % (rootName, rootVersion.replace('.', ''))
self.stepCount += 1
stepName = '%sStep%s' % (rootName, self.stepCount)
modulesNameList = [
'CreateDataFile', 'RootApplication', 'FileUsage',
'UserJobFinalization'
]
parametersList = [('rootVersion', 'string', '', 'Root version'),
('rootScript', 'string', '', 'Root script'),
('rootType', 'string', '', 'Root type'),
('arguments', 'list', [],
'Optional arguments for payload'),
('applicationLog', 'string', '', 'Log file name'),
('SystemConfig', 'string', '', 'binary tag')]
step = getStepDefinition(stepName,
modulesNameList=modulesNameList,
importLine="LHCbDIRAC.Workflow.Modules",
parametersList=parametersList)
logPrefix = 'Step%s_' % (self.stepCount)
logName = '%s%s' % (logPrefix, logName)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
# Define Step and its variables
stepInstance.setValue("rootVersion", rootVersion)
stepInstance.setValue("rootType", rootType)
stepInstance.setValue("rootScript", os.path.basename(rootScript))
stepInstance.setValue("applicationLog", logName)
stepInstance.setValue("SystemConfig", systemConfig)
if arguments:
stepInstance.setValue("arguments", arguments)
return S_OK(stepInstance)
def setApplicationScript(self,
appName,
appVersion,
script,
arguments='',
inputData='',
inputDataType='',
poolXMLCatalog='pool_xml_catalog.xml',
logFile='',
extraPackages='',
systemConfig='ANY',
modulesNameList=None,
parametersList=None):
"""Specifies application environment and script to be executed.
For LHCb these could be e.g. Gauss, Boole, Brunel,
DaVinci etc.
The script name and any arguments should also be specified.
Input data for application script must be specified here, please note that if this is set at the job level,
e.g. via setInputData() but not above, input data is not in the scope of the specified application.
Any input data specified at the step level that is not already specified at the job level is added automatically
as a requirement for the job.
Example usage:
>>> job = LHCbJob()
>>> job.setApplicationScript('DaVinci','v19r12','myScript.py')
:param appName: Application name
:type appName: string
:param appVersion: Application version
:type appVersion: string
:param script: Script to execute
:type script: string
:param arguments: Optional arguments for script
:type arguments: string
:param inputData: Input data for application
:type inputData: single LFN or list of LFNs
:param inputDataType: Input data type for application (e.g. DATA, MDF, ETC)
:type inputDataType: string
:param arguments: Optional POOL XML Catalog name for any input data files (default is pool_xml_catalog.xml)
:type arguments: string
:param logFile: Optional log file name
:type logFile: string
:param modulesNameList: list of module names. The default is given.
:type modulesNameList: list
:param parametersList: list of parameters. The default is given.
:type parametersList: list
"""
kwargs = {
'appName': appName,
'appVersion': appVersion,
'script': script,
'arguments': arguments,
'inputData': inputData,
'inputDataType': inputDataType,
'poolXMLCatalog': poolXMLCatalog,
'logFile': logFile
}
if not isinstance(appName, str) or not isinstance(appVersion, str):
return self._reportError(
'Expected strings for application name and version', __name__,
**kwargs)
if not script or not isinstance(script, str):
return self._reportError('Expected strings for script name',
__name__, **kwargs)
if not os.path.exists(script):
return self._reportError('Script must exist locally', __name__,
**kwargs)
if logFile:
if isinstance(logFile, str):
logName = logFile
else:
return self._reportError('Expected string for log file name',
__name__, **kwargs)
else:
shortScriptName = os.path.basename(script).split('.')[0]
logName = '%s_%s_%s.log' % (appName, appVersion, shortScriptName)
self.addToInputSandbox.append(script)
if arguments:
if not isinstance(arguments, str):
return self._reportError(
'Expected string for optional script arguments', __name__,
**kwargs)
if not isinstance(poolXMLCatalog, str):
return self._reportError(
'Expected string for POOL XML Catalog name', __name__,
**kwargs)
if inputData:
if isinstance(inputData, str):
inputData = [inputData]
if not isinstance(inputData, list):
return self._reportError(
'Expected single LFN string or list of LFN(s) for inputData',
__name__, **kwargs)
if inputData != ['previousStep']:
for i in xrange(len(inputData)):
inputData[i] = inputData[i].replace('LFN:', '')
inputData = ['LFN:' + x for x in inputData]
inputDataStr = ';'.join(inputData)
self.addToInputData.append(inputDataStr)
self.stepCount += 1
stepName = '%sStep%s' % (appName, self.stepCount)
if not modulesNameList:
modulesNameList = [
'CreateDataFile', 'GaudiApplicationScript', 'FileUsage',
'UserJobFinalization'
]
if not parametersList:
parametersList = [
('applicationName', 'string', '', 'Application Name'),
('applicationVersion', 'string', '', 'Application Version'),
('applicationLog', 'string', '',
'Name of the output file of the application'),
('script', 'string', '', 'Script Name'),
('arguments', 'string', '', 'Arguments for executable'),
('poolXMLCatName', 'string', '', 'POOL XML Catalog file name'),
('inputDataType', 'string', '', 'Input Data Type'),
('inputData', 'string', '', 'Input Data'),
('extraPackages', 'string', '', 'extraPackages'),
('SystemConfig', 'string', '', 'binary tag')
]
step = getStepDefinition(stepName,
modulesNameList=modulesNameList,
importLine="LHCbDIRAC.Workflow.Modules",
parametersList=parametersList)
logPrefix = 'Step%s_' % (self.stepCount)
logName = '%s%s' % (logPrefix, logName)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue("applicationName", appName)
stepInstance.setValue("applicationVersion", appVersion)
stepInstance.setValue("script", script)
stepInstance.setValue("applicationLog", logName)
if arguments:
stepInstance.setValue("arguments", arguments)
if inputDataType:
stepInstance.setValue("inputDataType", inputDataType)
if inputData:
stepInstance.setValue("inputData", ';'.join(inputData))
if poolXMLCatalog:
stepInstance.setValue("poolXMLCatName", poolXMLCatalog)
stepInstance.setValue('extraPackages', extraPackages)
stepInstance.setValue('SystemConfig', systemConfig)
return S_OK(stepInstance)
def setApplication(self,
appName,
appVersion,
optionsFiles,
inputData=None,
optionsLine=None,
inputDataType=None,
logFile=None,
events=-1,
extraPackages=None,
systemConfig='ANY',
modulesNameList=None,
parametersList=None):
"""Specifies gaudi application for DIRAC workflows, executed via gaudirun.
For LHCb these could be e.g. Gauss, Boole, Brunel, DaVinci, LHCb, etc.
The optionsFiles parameter can be the path to an options file or a list of paths to options files.
All options files are automatically appended to the job input sandBox but the first in the case of a
list is assumed to be the 'master' options file.
Input data for application script must be specified here, please note that if this is set at the job level,
e.g. via setInputData() but not above, input data is not in the scope of the specified application.
Any input data specified at the step level that is not already specified at the job level is added automatically
as a requirement for the job.
Example usage:
>>> job = LHCbJob()
>>> job.setApplication('DaVinci','v19r5',optionsFiles='MyDV.opts',
inputData=['/lhcb/production/DC06/phys-lumi2/00001501/DST/0000/00001501_00000320_5.dst'],logFile='dv.log')
:param appName: Application name
:type appName: string
:param appVersion: Application version
:type appVersion: string
:param optionsFiles: Path to options file(s) for application
:type optionsFiles: string or list
:param inputData: Input data for application (if a subset of the overall input data for a given job is required)
:type inputData: single LFN or list of LFNs
:param optionsLine: Additional options lines for application
:type optionsLine: string
:param inputDataType: Input data type for application (e.g. DATA, MDF, ETC)
:type inputDataType: string
:param logFile: Optional log file name
:type logFile: string
:param events: Optional number of events
:type events: integer
:param extraPackages: Optional extra packages
:type extraPackages: string
:param modulesNameList: list of module names. The default is given.
:type modulesNameList: list
:param parametersList: list of parameters. The default is given.
:type parametersList: list
"""
kwargs = {
'appName': appName,
'appVersion': appVersion,
'optionsFiles': optionsFiles,
'inputData': inputData,
'optionsLine': optionsLine,
'inputDataType': inputDataType,
'logFile': logFile
}
if not isinstance(appName, str) or not isinstance(appVersion, str):
return self._reportError(
'Expected strings for application name and version', __name__,
**kwargs)
if logFile:
if isinstance(logFile, str):
logName = logFile
else:
return self._reportError('Expected string for log file name',
__name__, **kwargs)
else:
logName = '%s_%s.log' % (appName, appVersion)
if not inputDataType:
inputDataType = self.inputDataType
if not isinstance(inputDataType, str):
return self._reportError('Expected string for input data type',
__name__, **kwargs)
optionsFile = None
if not optionsFiles:
return self._reportError(
'Expected string or list for optionsFiles', __name__, **kwargs)
if isinstance(optionsFiles, str):
optionsFiles = [optionsFiles]
if not isinstance(optionsFiles, list):
return self._reportError(
'Expected string or list for optionsFiles', __name__, **kwargs)
for optsFile in optionsFiles:
if not optionsFile:
self.log.verbose('Found master options file %s' % optsFile)
optionsFile = optsFile
if os.path.exists(optsFile):
self.log.verbose('Found specified options file: %s' % optsFile)
self.addToInputSandbox.append(optsFile)
optionsFile += ';%s' % optsFile
elif re.search(r'\$', optsFile):
self.log.verbose(
'Assuming %s is using an environment variable to be resolved during execution'
% optsFile)
if not optionsFile == optsFile:
optionsFile += ';%s' % optsFile
else:
return self._reportError(
'Specified options file %s does not exist' % (optsFile),
__name__, **kwargs)
# ensure optionsFile list is unique:
tmpList = optionsFile.split(';')
optionsFile = ';'.join(list(set(tmpList)))
self.log.verbose('Final options list is: %s' % optionsFile)
if inputData:
if isinstance(inputData, str):
inputData = [inputData]
if not isinstance(inputData, list):
return self._reportError(
'Expected single LFN string or list of LFN(s) for inputData',
__name__, **kwargs)
if inputData != ['previousStep']:
for i in xrange(len(inputData)):
inputData[i] = inputData[i].replace('LFN:', '')
inputData = ['LFN:' + x for x in inputData]
inputDataStr = ';'.join(inputData)
self.addToInputData.append(inputDataStr)
self.stepCount += 1
stepName = '%sStep%s' % (appName, self.stepCount)
if not modulesNameList:
modulesNameList = [
'CreateDataFile', 'GaudiApplication', 'FileUsage',
'AnalyseFileAccess', 'UserJobFinalization'
]
if not parametersList:
parametersList = [
('applicationName', 'string', '', 'Application Name'),
('applicationVersion', 'string', '', 'Application Version'),
('applicationLog', 'string', '',
'Name of the output file of the application'),
('optionsFile', 'string', '', 'Options File'),
('extraOptionsLine', 'string', '',
'This is appended to standard options'),
('inputDataType', 'string', '', 'Input Data Type'),
('inputData', 'string', '', 'Input Data'),
('numberOfEvents', 'string', '', 'Events treated'),
('extraPackages', 'string', '', 'ExtraPackages'),
('SystemConfig', 'string', '', 'binary tag')
]
step = getStepDefinition(stepName,
modulesNameList=modulesNameList,
importLine="LHCbDIRAC.Workflow.Modules",
parametersList=parametersList)
logPrefix = 'Step%s_' % (self.stepCount)
logName = '%s%s' % (logPrefix, logName)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue('applicationName', appName)
stepInstance.setValue('applicationVersion', appVersion)
stepInstance.setValue('applicationLog', logName)
if optionsFile:
stepInstance.setValue('optionsFile', optionsFile)
if optionsLine:
stepInstance.setValue('extraOptionsLine', optionsLine)
if inputDataType:
stepInstance.setValue('inputDataType', inputDataType)
if inputData:
stepInstance.setValue('inputData', ';'.join(inputData))
stepInstance.setValue('numberOfEvents', str(events))
stepInstance.setValue('extraPackages', extraPackages)
stepInstance.setValue('SystemConfig', systemConfig)
return S_OK(stepInstance)
def setExecutable( self, executable, arguments = '', logFile = '',
modulesList = None, parameters = None, paramValues = None ):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
:param executable: Executable
:type executable: string
:param arguments: Optional arguments to executable
:type arguments: string
:param logFile: Optional log file name
:type logFile: string
:param modulesList: Optional list of modules (to be used mostly when extending this method)
:type modulesList: list
:param parameters: Optional list of parameters (to be used mostly when extending this method)
:type parameters: list of tuples
:param paramValues: Optional list of parameters values (to be used mostly when extending this method)
:type parameters: list of tuples
"""
kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
if not isinstance( executable, basestring ) or not isinstance( arguments, basestring ) or \
not isinstance( logFile, basestring ):
return self._reportError( 'Expected strings for executable and arguments', **kwargs )
if os.path.exists( executable ):
self.log.verbose( 'Found script executable file %s' % ( executable ) )
self.addToInputSandbox.append( executable )
logName = '%s.log' % ( os.path.basename( executable ) )
else:
self.log.warn( 'The executable code could not be found locally' )
logName = 'CodeOutput.log'
self.stepCount += 1
stepName = 'RunScriptStep%s' % ( self.stepCount )
if logFile:
if isinstance( logFile, basestring ):
logName = str(logFile)
else:
logName = "Script%s_%s" %( self.stepCount, logName )
if not modulesList:
modulesList = ['Script']
if not parameters:
parameters = [( 'executable', 'string', '', "Executable Script" ),
( 'arguments', 'string', '', 'Arguments for executable Script' ),
( 'applicationLog', 'string', '', "Log file name" )]
step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
self.addToOutputSandbox.append( logName )
stepInstance = addStepToWorkflow( self.workflow, step, stepName )
stepInstance.setValue( 'applicationLog', logName )
stepInstance.setValue( 'executable', executable )
if arguments:
# If arguments are expressed in terms of parameters, pass them to Workflow
# These arguments will be resolved on the server side for each parametric job
if re.search( '%\(.*\)s', arguments ) or re.search( '%n', arguments ):
self.parametricWFArguments['arguments'] = arguments
else:
stepInstance.setValue( 'arguments', arguments )
if paramValues:
for param, value in paramValues:
stepInstance.setValue( param, value )
return S_OK( stepInstance )
def setExecutable( self, executable, arguments = '', logFile = '',
modulesList = ['Script'],
parameters = [( 'executable', 'string', '', "Executable Script" ),
( 'arguments', 'string', '', 'Arguments for executable Script' ),
( 'applicationLog', 'string', '', "Log file name" )],
paramValues = [] ):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
@param executable: Executable
@type executable: string
@param arguments: Optional arguments to executable
@type arguments: string
@param logFile: Optional log file name
@type logFile: string
@param modulesList: Optional list of modules (to be used mostly when extending this method)
@type modulesList: list
@param parameters: Optional list of parameters (to be used mostly when extending this method)
@type parameters: list of tuples
@param paramValues: Optional list of parameters values (to be used mostly when extending this method)
@type parameters: list of tuples
"""
kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or \
not type( logFile ) == type( ' ' ):
return self._reportError( 'Expected strings for executable and arguments', **kwargs )
if os.path.exists( executable ):
self.log.verbose( 'Found script executable file %s' % ( executable ) )
self.addToInputSandbox.append( executable )
logName = '%s.log' % ( os.path.basename( executable ) )
else:
self.log.warn( 'The executable code could not be found locally' )
logName = 'CodeOutput.log'
self.stepCount += 1
stepName = 'RunScriptStep%s' % ( self.stepCount )
if logFile:
if type( logFile ) == type( ' ' ):
logName = str(logFile)
else:
logName = "Script%s_%s" %( self.stepCount, logName )
step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
self.addToOutputSandbox.append( logName )
stepInstance = addStepToWorkflow( self.workflow, step, stepName )
stepInstance.setValue( 'applicationLog', logName )
stepInstance.setValue( 'executable', executable )
if arguments:
stepInstance.setValue( 'arguments', arguments )
if paramValues:
for param, value in paramValues:
stepInstance.setValue( param, value )
return S_OK( stepInstance )
def addApplicationStep(self, stepDict, inputData=None, modulesList=None):
""" Adds an application step to the workflow
stepDict contains everything that is in the step, for this production, e.g.::
{'ApplicationName': 'DaVinci', 'Usable': 'Yes', 'StepId': 13718, 'ApplicationVersion': 'v28r3p1',
'ExtraPackages': 'AppConfig.v3r104', 'StepName': 'Stripping14-Merging', 'ExtraOptions': '',
'ProcessingPass': '******', 'Visible': 'N', 'OptionsFormat': '',
'OptionFiles': '$APPCONFIGOPTS/Merging/DV-Stripping14-Merging.py',
'DDDB': 'head-20110302', 'CONDDB': 'head-20110407', 'DQTag': '',
'isMulticore': 'N', 'SystemConfig': '', 'mcTCK': '',
'fileTypesIn': ['SDST'],
'visibilityFlag': [{'Visible': 'Y', 'FileType': 'BHADRON.DST'}],
'fileTypesOut': ['BHADRON.DST', 'CALIBRATION.DST', 'CHARM.MDST', 'CHARMCOMPLETEEVENT.DST']}
Note: this step treated here does not necessarily corresponds to a step of the BKK:
the case where they might be different is the merging case.
:param dict stepDict: contains everything that is in the step, for this production
:param str inputData: the input data of the step. Either None, or 'previousStep', or the input to the step
:param list modulesList: the list of module names (str) this step is made of. If None, a default is taken.
:returns: the name (str) of the step added
"""
appName = stepDict['ApplicationName']
appVersion = stepDict['ApplicationVersion']
optionsFile = stepDict['OptionFiles']
stepID = stepDict['StepId']
stepName = stepDict['StepName']
stepVisible = stepDict['Visible']
extraPackages = stepDict['ExtraPackages']
optionsLine = stepDict['ExtraOptions']
fileTypesIn = stepDict['fileTypesIn']
fileTypesOut = stepDict['fileTypesOut']
stepPass = stepDict['ProcessingPass']
optionsFormat = stepDict['OptionsFormat']
dddbOpt = stepDict['DDDB']
conddbOpt = stepDict['CONDDB']
dqOpt = stepDict['DQTag']
multicore = stepDict['isMulticore']
sysConfig = stepDict['SystemConfig']
outputVisibility = stepDict['visibilityFlag']
if sysConfig == 'None' or sysConfig == 'NULL' or not sysConfig or sysConfig is None:
sysConfig = 'ANY'
mcTCK = stepDict['mcTCK']
if extraPackages:
if isinstance(extraPackages, list):
extraPackages = ';'.join(extraPackages)
if not isinstance(extraPackages, str):
raise TypeError("extraPackages is not a string (nor a list)")
if ',' in extraPackages:
extraPackages = ';'.join(extraPackages.split(','))
if 'ProdConf' not in extraPackages:
extraPackages = extraPackages + ';ProdConf'
extraPackages = extraPackages.replace(' ', '')
else:
extraPackages = 'ProdConf'
self.__checkArguments(extraPackages, optionsFile)
try:
if not dddbOpt.lower() == 'global':
gLogger.verbose(
"Specific DDDBTag setting found for %s step, setting to: %s"
% (appName, dddbOpt))
dddbOpt = dddbOpt.replace(' ', '')
except AttributeError:
pass
try:
if not conddbOpt.lower() == 'global':
gLogger.verbose(
"Specific CondDBTag setting found for %s step, setting to: %s"
% (appName, conddbOpt))
conddbOpt = conddbOpt.replace(' ', '')
except AttributeError:
pass
try:
if not dqOpt.lower() == 'global':
gLogger.verbose(
"Specific DQTag setting found for %s step, setting to: %s"
% (appName, dqOpt))
dqOpt = dqOpt.replace(' ', '')
except AttributeError:
pass
# starting real stuff
self.LHCbJob.stepCount += 1
if 'Gaudi_App_Step' not in self.LHCbJob.workflow.step_definitions.keys(
):
gLogger.debug(
"Determining the modules of the steps (modulesList = %s)" %
modulesList)
if modulesList is None: # we assume it's a standard list of modules for Gaudi steps
gaudiPath = 'Productions/GaudiStep_Modules'
modulesList = self.opsHelper.getValue(gaudiPath, [
'GaudiApplication', 'AnalyseXMLSummary', 'ErrorLogging',
'BookkeepingReport', 'StepAccounting'
])
# pName, pType, pValue, pDesc
parametersList = [
['inputData', 'string', '', 'StepInputData'],
['inputDataType', 'string', '', 'InputDataType'],
['applicationName', 'string', '', 'ApplicationName'],
['applicationVersion', 'string', '', 'ApplicationVersion'],
['runTimeProjectName', 'string', '', 'runTimeProjectName'],
[
'runTimeProjectVersion', 'string', '',
'runTimeProjectVersion'
], ['applicationType', 'string', '', 'ApplicationType'],
['optionsFile', 'string', '', 'OptionsFile'],
['extraOptionsLine', 'string', '', 'extraOptionsLines'],
['listoutput', 'list', [], 'StepOutputList'],
['extraPackages', 'string', '', 'ExtraPackages'],
['BKStepID', 'string', '', 'BKKStepID'],
['StepProcPass', 'string', '', 'StepProcessingPass'],
['optionsFormat', 'string', '', 'ProdConf configuration'],
['CondDBTag', 'string', '', 'ConditionDatabaseTag'],
['DDDBTag', 'string', '', 'DetDescTag'],
['DQTag', 'string', '', 'DataQualityTag'],
['multiCore', 'string', '', 'MultiCore Flag'],
['SystemConfig', 'string', '', 'system config'],
['mcTCK', 'string', '', 'TCK to be simulated']
]
gaudiStepDef = getStepDefinition(
'Gaudi_App_Step',
modulesNameList=modulesList,
importLine='LHCbDIRAC.Workflow.Modules',
parametersList=parametersList)
self.LHCbJob.workflow.addStep(gaudiStepDef)
# create the step instance add it to the wf, and return it
name = '%s_%s' % (appName, self.LHCbJob.stepCount)
gaudiStepInstance = self.LHCbJob.workflow.createStepInstance(
'Gaudi_App_Step', name)
valuesToSet = [['applicationName', appName],
['applicationVersion', appVersion],
['optionsFile', optionsFile],
['extraOptionsLine', optionsLine],
['extraPackages', extraPackages],
['BKStepID', str(stepID)], ['StepProcPass', stepPass],
['optionsFormat', optionsFormat],
['CondDBTag', conddbOpt], ['DDDBTag', dddbOpt],
['DQTag', dqOpt], ['multiCore', multicore],
['SystemConfig', sysConfig], ['mcTCK', mcTCK]]
if fileTypesIn:
valuesToSet.append([
'inputDataType', ';'.join(ftIn.upper() for ftIn in fileTypesIn)
])
if inputData is None:
gLogger.verbose(
'%s step has no data requirement or is linked to the overall input data'
% appName)
gaudiStepInstance.setLink('inputData', 'self', 'InputData')
elif inputData == 'previousStep':
gLogger.verbose(
'Taking input data as output from previous Gaudi step')
valuesToSet.append(['inputData', inputData])
else:
gLogger.verbose(
'Assume input data requirement should be added to job')
self.LHCbJob.setInputData(inputData)
gaudiStepInstance.setLink('inputData', 'self', 'InputData')
for pName, value in valuesToSet:
if value:
gaudiStepInstance.setValue(pName, value)
outputFilesList = self._constructOutputFilesList(fileTypesOut)
gaudiStepInstance.setValue('listoutput', (outputFilesList))
# to construct the BK processing pass structure, starts from step '0'
stepIDInternal = 'Step%s' % (self.LHCbJob.stepCount - 1)
bkOptionsFile = optionsFile
stepBKInfo = {
'ApplicationName': appName,
'ApplicationVersion': appVersion,
'OptionFiles': bkOptionsFile,
'DDDb': dddbOpt,
'CondDb': conddbOpt,
'DQTag': dqOpt,
'ExtraPackages': extraPackages,
'BKStepID': stepID,
'StepName': stepName,
'StepVisible': stepVisible,
'OutputFileTypes': outputVisibility
}
self.bkSteps[stepIDInternal] = stepBKInfo
self.__addBKPassStep()
return name
def setExecutable(self, executable, arguments="", logFile="", modulesList=None, parameters=None, paramValues=None):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
@param executable: Executable
@type executable: string
@param arguments: Optional arguments to executable
@type arguments: string
@param logFile: Optional log file name
@type logFile: string
@param modulesList: Optional list of modules (to be used mostly when extending this method)
@type modulesList: list
@param parameters: Optional list of parameters (to be used mostly when extending this method)
@type parameters: list of tuples
@param paramValues: Optional list of parameters values (to be used mostly when extending this method)
@type parameters: list of tuples
"""
kwargs = {"executable": executable, "arguments": arguments, "logFile": logFile}
if not type(executable) == type(" ") or not type(arguments) == type(" ") or not type(logFile) == type(" "):
return self._reportError("Expected strings for executable and arguments", **kwargs)
if os.path.exists(executable):
self.log.verbose("Found script executable file %s" % (executable))
self.addToInputSandbox.append(executable)
logName = "%s.log" % (os.path.basename(executable))
else:
self.log.warn("The executable code could not be found locally")
logName = "CodeOutput.log"
self.stepCount += 1
stepName = "RunScriptStep%s" % (self.stepCount)
if logFile:
if type(logFile) == type(" "):
logName = str(logFile)
else:
logName = "Script%s_%s" % (self.stepCount, logName)
if not modulesList:
modulesList = ["Script"]
if not parameters:
parameters = [
("executable", "string", "", "Executable Script"),
("arguments", "string", "", "Arguments for executable Script"),
("applicationLog", "string", "", "Log file name"),
]
step = getStepDefinition("ScriptStep%s" % (self.stepCount), modulesList, parametersList=parameters)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue("applicationLog", logName)
stepInstance.setValue("executable", executable)
if arguments:
stepInstance.setValue("arguments", arguments)
if paramValues:
for param, value in paramValues:
stepInstance.setValue(param, value)
return S_OK(stepInstance)
def setExecutable(self, executable, arguments='', logFile='',
modulesList=None, parameters=None, paramValues=None):
"""Helper function.
Specify executable script to run with optional arguments and log file
for standard output.
These can be either:
- Submission of a python or shell script to DIRAC
- Can be inline scripts e.g. C{'/bin/ls'}
- Scripts as executables e.g. python or shell script file
Example usage:
>>> job = Job()
>>> job.setExecutable('myScript.py')
:param executable: Executable
:type executable: string
:param arguments: Optional arguments to executable
:type arguments: string
:param logFile: Optional log file name
:type logFile: string
:param modulesList: Optional list of modules (to be used mostly when extending this method)
:type modulesList: python:list
:param parameters: Optional list of parameters (to be used mostly when extending this method)
:type parameters: python:list of tuples
:param paramValues: Optional list of parameters values (to be used mostly when extending this method)
:type parameters: python:list of tuples
"""
kwargs = {'executable': executable, 'arguments': arguments, 'logFile': logFile}
if not isinstance(executable, basestring) or not isinstance(arguments, six.string_types) or \
not isinstance(logFile, six.string_types):
return self._reportError('Expected strings for executable and arguments', **kwargs)
if os.path.exists(executable):
self.log.verbose('Found script executable file %s' % (executable))
self.addToInputSandbox.append(executable)
logName = '%s.log' % (os.path.basename(executable))
else:
self.log.warn('The executable code could not be found locally')
logName = 'CodeOutput.log'
self.stepCount += 1
stepName = 'RunScriptStep%s' % (self.stepCount)
if logFile:
if isinstance(logFile, six.string_types):
logName = str(logFile)
else:
logName = "Script%s_%s" % (self.stepCount, logName)
if not modulesList:
modulesList = ['Script']
if not parameters:
parameters = [('executable', 'string', '', "Executable Script"),
('arguments', 'string', '', 'Arguments for executable Script'),
('applicationLog', 'string', '', "Log file name")]
step = getStepDefinition('ScriptStep%s' % (self.stepCount), modulesList, parametersList=parameters)
self.addToOutputSandbox.append(logName)
stepInstance = addStepToWorkflow(self.workflow, step, stepName)
stepInstance.setValue('applicationLog', logName)
stepInstance.setValue('executable', executable)
if arguments:
# If arguments are expressed in terms of parameters, pass them to Workflow
# These arguments will be resolved on the server side for each parametric job
if re.search(r'%\(.*\)s', arguments) or re.search('%n', arguments):
self.parametricWFArguments['arguments'] = arguments
else:
stepInstance.setValue('arguments', arguments)
if paramValues:
for param, value in paramValues:
stepInstance.setValue(param, value)
return S_OK(stepInstance)