def createSingleModuleWorkflow(module, name):
''' Creates a workflow based on a single module definition
'''
moduleType = module.getType()
moduleName = name
workflow = Workflow()
step = StepDefinition(moduleType + '_step')
step.addModule(module)
moduleInstance = step.createModuleInstance(moduleType, moduleName)
step.addParameter(moduleInstance.parameters.getInput())
workflow.addParameter(moduleInstance.parameters.getInput())
workflow.addStep(step)
stepInstance = workflow.createStepInstance(moduleType + '_step',
moduleName + '_step')
# Propagate the module input parameters to the workflow level
moduleInstance.linkParameterUp(moduleInstance.parameters.getInput())
stepInstance.linkParameterUp(moduleInstance.parameters.getInput())
workflow.setName(name)
workflow.setDescription('Single module workflow from ' + moduleType +
' type module')
workflow.setDescrShort(moduleType + ' workflow')
return workflow
def addWrapper(self, logFile=''):
""" Overload the DIRAC.Job.setExecutable
"""
logFile = str(logFile)
stepDefn = 'WrapperStep'
stepName = 'RunWrapperStep'
moduleName = 'GlastWrapperCall'
module = ModuleDefinition(moduleName)
module.setDescription('The utility that calls the pipeline_wrapper.')
body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
module.setBody(body)
# Create Step definition
step = StepDefinition(stepDefn)
step.addModule(module)
moduleInstance = step.createModuleInstance('GlastWrapperCall',
stepDefn)
# Define step parameters
step.addParameter(
Parameter("logFile", "", "string", "", "", False, False,
'Log file name'))
self.addToOutputSandbox.append(logFile)
self.workflow.addStep(step)
# Define Step and its variables
stepInstance = self.workflow.createStepInstance(stepDefn, stepName)
stepInstance.setValue("logFile", logFile)
return S_OK()
def _addRealFinalization(self):
""" This is called at creation: now that the workflow is created at the last minute,
we need to add this also at the last minute
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest, 'enable', 'bool', False,
'EnableFlag')
body = importLine.replace('<MODULE>', 'FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('RegisterOutputData')
registerdata.setDescription(
'Module to add in the metadata catalog the relevant info about the files'
)
self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'RegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadLogFile')
logUpload.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData',
'dataUpload')
up.setValue("enable", self.finalsdict['uploadData'])
finalization.addModule(registerdata)
ro = finalization.createModuleInstance('RegisterOutputData',
'RegisterOutputData')
ro.setValue("enable", self.finalsdict['registerData'])
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
ul.setValue("enable", self.finalsdict['uploadLog'])
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest',
'failoverRequest')
fr.setValue("enable", self.finalsdict['sendFailover'])
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
def _addRealFinalization(self):
""" See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest, 'enable', 'bool', False,
'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('ILDRegisterOutputData')
registerdata.setDescription(
'Module to add in the metadata catalog the relevant info about the files'
)
self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'ILDRegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'UploadLogFile')
logUpload.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData',
'dataUpload')
up.setValue("enable", self.finalsdict['uploadData'])
finalization.addModule(registerdata)
# TODO: create ILDRegisterOutputData
ro = finalization.createModuleInstance('ILDRegisterOutputData',
'ILDRegisterOutputData')
ro.setValue("enable", self.finalsdict['registerData'])
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
ul.setValue("enable", self.finalsdict['uploadLog'])
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest',
'failoverRequest')
fr.setValue("enable", self.finalsdict['sendFailover'])
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
def getStepDefinition(stepName,
modulesNameList=None,
importLine="""""",
parametersList=None):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition(moduleName)
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (importLine, moduleName), globals(),
locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(importLine, moduleName, moduleName))
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (alternativeImportLine, moduleName),
globals(), locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(alternativeImportLine, moduleName, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def test__getStepDefinition(self):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition("Script")
body = importLine.replace("<MODULE>", "Script")
gaudiApp.setDescription(
getattr(
__import__("%s.%s" % ("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), "__doc__"
)
)
gaudiApp.setBody(body)
genBKReport = ModuleDefinition("FailoverRequest")
body = importLine.replace("<MODULE>", "FailoverRequest")
genBKReport.setDescription(
getattr(
__import__("%s.%s" % ("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"]),
"__doc__",
)
)
genBKReport.setBody(body)
# step
appDefn = StepDefinition("App_Step")
appDefn.addModule(gaudiApp)
appDefn.createModuleInstance("Script", "Script")
appDefn.addModule(genBKReport)
appDefn.createModuleInstance("FailoverRequest", "FailoverRequest")
appDefn.addParameterLinked(gaudiApp.parameters)
stepDef = getStepDefinition("App_Step", ["Script", "FailoverRequest"])
self.assertTrue(str(appDefn) == str(stepDef))
self.job._addParameter(appDefn, "name", "type", "value", "desc")
self.job._addParameter(appDefn, "name1", "type1", "value1", "desc1")
stepDef = getStepDefinition(
"App_Step",
["Script", "FailoverRequest"],
parametersList=[["name", "type", "value", "desc"], ["name1", "type1", "value1", "desc1"]],
)
self.assertTrue(str(appDefn) == str(stepDef))
def startElement(self, name, attrs):
# print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames()
self.clearCharacters(
) # clear to remove empty or nonprintable characters
if name == "Workflow":
if self.root == None: # if root not defined by constractor
self.root = Workflow()
self.stack.append(self.root)
elif name == "StepDefinition":
obj = StepDefinition("TemporaryXMLObject_StepDefinition")
if self.root == None: # in case we are saving Step only
self.root = obj
self.stack.append(obj)
elif name == "StepInstance":
obj = StepInstance("TemporaryXMLObject_StepInstance")
self.stack.append(obj)
elif name == "ModuleDefinition":
obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition")
if self.root == None: # in case we are saving Module only
self.root = obj
self.stack.append(obj)
elif name == "ModuleInstance":
obj = ModuleInstance("TemporaryXMLObject_ModuleInstance")
self.stack.append(obj)
elif name == "Parameter":
obj = Parameter(str(attrs['name']), None, str(attrs['type']),
str(attrs['linked_module']),
str(attrs['linked_parameter']), str(attrs['in']),
str(attrs['out']), str(attrs['description']))
self.stack.append(obj)
# TEMPORARY CODE
elif name == "origin" or name == "version" or name == "name" or name == "type" or name == "value" or\
name == "required" or name == "descr_short" or name == "name" or name == "type" or name == "description" or name == "body":
pass
else:
print "UNTREATED! startElement name=", name, "attr=", attrs.getLength(
), attrs.getNames()
pass
def test__getStepDefinition(self):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition('Script')
body = importLine.replace('<MODULE>', 'Script')
gaudiApp.setDescription(
getattr(
__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'Script'),
globals(), locals(), ['__doc__']), "__doc__"))
gaudiApp.setBody(body)
genBKReport = ModuleDefinition('FailoverRequest')
body = importLine.replace('<MODULE>', 'FailoverRequest')
genBKReport.setDescription(
getattr(
__import__(
"%s.%s" % ('DIRAC.Workflow.Modules', 'FailoverRequest'),
globals(), locals(), ['__doc__']), "__doc__"))
genBKReport.setBody(body)
# step
appDefn = StepDefinition('App_Step')
appDefn.addModule(gaudiApp)
appDefn.createModuleInstance('Script', 'Script')
appDefn.addModule(genBKReport)
appDefn.createModuleInstance('FailoverRequest', 'FailoverRequest')
appDefn.addParameterLinked(gaudiApp.parameters)
stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'])
self.assert_(str(appDefn) == str(stepDef))
self.job._addParameter(appDefn, 'name', 'type', 'value', 'desc')
self.job._addParameter(appDefn, 'name1', 'type1', 'value1', 'desc1')
stepDef = getStepDefinition(
'App_Step', ['Script', 'FailoverRequest'],
parametersList=[['name', 'type', 'value', 'desc'],
['name1', 'type1', 'value1', 'desc1']])
self.assert_(str(appDefn) == str(stepDef))
def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step
def getStepDefinition(stepName,
modulesNameList=None,
importLine="",
parametersList=None):
"""Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
module = None
if importLine:
try:
module = importlib.import_module(importLine + "." + moduleName)
except ImportError:
pass
# In case the importLine is not set, this is looking for a DIRAC extension, if any
if module is None:
module = ObjectLoader().loadModule("Workflow.Modules." +
moduleName)["Value"]
# create the module definition
moduleDef = ModuleDefinition(moduleName)
moduleDef.setDescription(module.__doc__)
moduleDef.setBody("\nfrom %s import %s\n" %
(module.__name__, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def _addToWorkflow(self):
""" This is called just before submission. It creates the actual workflow.
The linking of parameters can only be done here
"""
for application in self.applicationlist:
#Start by defining step number
self.stepCount += 1
res = application._analyseJob(self)
if not res['OK']:
return res
res = application._checkWorkflowConsistency()
if not res['OK']:
self.log.error(
"%s failed to check its consistency:" % application,
"%s" % res['Message'])
return S_ERROR("%s failed to check its consistency: %s" %
(application, res['Message']))
##Now we can create the step and add it to the workflow
#First we need a unique name, let's use the application name and step number
stepname = "%s_step_%s" % (application.appname, self.stepCount)
stepdefinition = StepDefinition(stepname)
self.steps.append(stepdefinition)
##Set the modules needed by the application
res = self._jobSpecificModules(application, stepdefinition)
if not res['OK']:
self.log.error("Failed to add modules:", "%s" % res['Message'])
return S_ERROR("Failed to add modules: %s" % res['Message'])
### add the parameters to the step
res = application._addParametersToStep(stepdefinition)
if not res['OK']:
self.log.error("Failed to add parameters:",
"%s" % res['Message'])
return S_ERROR("Failed to add parameters: %s" % res['Message'])
##Now the step is defined, let's add it to the workflow
self.workflow.addStep(stepdefinition)
###Now we need to get a step instance object to set the parameters' values
stepInstance = self.workflow.createStepInstance(
stepdefinition.getType(), stepname)
##Set the parameters values to the step instance
res = application._setStepParametersValues(stepInstance)
if not res['OK']:
self.log.error("Failed to resolve parameters values:",
"%s" % res['Message'])
return S_ERROR("Failed to resolve parameters values: %s" %
res['Message'])
res = application._resolveLinkedStepParameters(stepInstance)
if not res['OK']:
self.log.error("Failed to resolve linked parameters:",
"%s" % res['Message'])
return S_ERROR("Failed to resolve linked parameters: %s" %
res['Message'])
#Now prevent overwriting of parameter values.
application._addedtojob()
self._addParameter(self.workflow, 'TotalSteps', 'String',
self.stepCount, 'Total number of steps')
return S_OK()
