def addWrapper(self, logFile = ''):
""" Overload the DIRAC.Job.setExecutable
"""
logFile = str(logFile)
stepDefn = 'WrapperStep'
stepName = 'RunWrapperStep'
moduleName = 'GlastWrapperCall'
module = ModuleDefinition( moduleName )
module.setDescription( 'The utility that calls the pipeline_wrapper.' )
body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
module.setBody( body )
# Create Step definition
step = StepDefinition( stepDefn )
step.addModule( module )
moduleInstance = step.createModuleInstance( 'GlastWrapperCall', stepDefn )
# Define step parameters
step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
self.addToOutputSandbox.append( logFile )
self.workflow.addStep( step )
# Define Step and its variables
stepInstance = self.workflow.createStepInstance( stepDefn, stepName )
stepInstance.setValue( "logFile", logFile )
return S_OK()
def addWrapper(self, logFile=''):
""" Overload the DIRAC.Job.setExecutable
"""
logFile = str(logFile)
stepDefn = 'WrapperStep'
stepName = 'RunWrapperStep'
moduleName = 'GlastWrapperCall'
module = ModuleDefinition(moduleName)
module.setDescription('The utility that calls the pipeline_wrapper.')
body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
module.setBody(body)
# Create Step definition
step = StepDefinition(stepDefn)
step.addModule(module)
moduleInstance = step.createModuleInstance('GlastWrapperCall',
stepDefn)
# Define step parameters
step.addParameter(
Parameter("logFile", "", "string", "", "", False, False,
'Log file name'))
self.addToOutputSandbox.append(logFile)
self.workflow.addStep(step)
# Define Step and its variables
stepInstance = self.workflow.createStepInstance(stepDefn, stepName)
stepInstance.setValue("logFile", logFile)
return S_OK()
def createSingleModuleWorkflow(module, name):
''' Creates a workflow based on a single module definition
'''
moduleType = module.getType()
moduleName = name
workflow = Workflow()
step = StepDefinition(moduleType + '_step')
step.addModule(module)
moduleInstance = step.createModuleInstance(moduleType, moduleName)
step.addParameter(moduleInstance.parameters.getInput())
workflow.addParameter(moduleInstance.parameters.getInput())
workflow.addStep(step)
stepInstance = workflow.createStepInstance(moduleType + '_step',
moduleName + '_step')
# Propagate the module input parameters to the workflow level
moduleInstance.linkParameterUp(moduleInstance.parameters.getInput())
stepInstance.linkParameterUp(moduleInstance.parameters.getInput())
workflow.setName(name)
workflow.setDescription('Single module workflow from ' + moduleType +
' type module')
workflow.setDescrShort(moduleType + ' workflow')
return workflow
def createSingleModuleWorkflow(module,name):
''' Creates a workflow based on a single module definition
'''
moduleType = module.getType()
moduleName = name
workflow = Workflow()
step = StepDefinition(moduleType+'_step')
step.addModule(module)
moduleInstance = step.createModuleInstance(moduleType,moduleName)
step.addParameter(moduleInstance.parameters.getInput())
workflow.addParameter(moduleInstance.parameters.getInput())
workflow.addStep(step)
stepInstance = workflow.createStepInstance(moduleType+'_step',moduleName+'_step')
# Propagate the module input parameters to the workflow level
moduleInstance.linkParameterUp(moduleInstance.parameters.getInput())
stepInstance.linkParameterUp(moduleInstance.parameters.getInput())
workflow.setName(name)
workflow.setDescription('Single module workflow from '+moduleType+' type module')
workflow.setDescrShort(moduleType+' workflow')
return workflow
def getStepDefinition(stepName,
modulesNameList=None,
importLine="""""",
parametersList=None):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition(moduleName)
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (importLine, moduleName), globals(),
locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(importLine, moduleName, moduleName))
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (alternativeImportLine, moduleName),
globals(), locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(alternativeImportLine, moduleName, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def getStepDefinition(stepName,
modulesNameList=None,
importLine="",
parametersList=None):
"""Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
module = None
if importLine:
try:
module = importlib.import_module(importLine + "." + moduleName)
except ImportError:
pass
# In case the importLine is not set, this is looking for a DIRAC extension, if any
if module is None:
module = ObjectLoader().loadModule("Workflow.Modules." +
moduleName)["Value"]
# create the module definition
moduleDef = ModuleDefinition(moduleName)
moduleDef.setDescription(module.__doc__)
moduleDef.setBody("\nfrom %s import %s\n" %
(module.__name__, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def getStepDefinition( stepName, modulesNameList = [], importLine = """""", parametersList = [] ):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition( stepName )
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition( moduleName )
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription( getattr( __import__( "%s.%s" % ( importLine, moduleName ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( importLine, moduleName, moduleName ) )
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription( getattr( __import__( "%s.%s" % ( alternativeImportLine, moduleName ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( alternativeImportLine, moduleName, moduleName ) )
# add the module to the step, and instance it
stepDef.addModule( moduleDef )
stepDef.createModuleInstance( module_type = moduleName, name = moduleName )
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter( pName, pValue, pType, "", "", True, False, pDesc )
stepDef.addParameter( Parameter( parameter = p ) )
return stepDef
def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step
def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step
