def addWrapper(self, logFile = ''):
""" Overload the DIRAC.Job.setExecutable
"""
logFile = str(logFile)
stepDefn = 'WrapperStep'
stepName = 'RunWrapperStep'
moduleName = 'GlastWrapperCall'
module = ModuleDefinition( moduleName )
module.setDescription( 'The utility that calls the pipeline_wrapper.' )
body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
module.setBody( body )
# Create Step definition
step = StepDefinition( stepDefn )
step.addModule( module )
#moduleInstance = step.createModuleInstance( 'GlastWrapperCall', stepDefn )
# Define step parameters
step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
self.addToOutputSandbox.append( logFile )
self.workflow.addStep( step )
# Define Step and its variables
stepInstance = self.workflow.createStepInstance( stepDefn, stepName )
stepInstance.setValue( "logFile", logFile )
return S_OK()
def _addToWorkflow(self):
""" This is called just before submission. It creates the actual workflow.
The linking of parameters can only be done here
"""
for application in self.applicationlist:
#Start by defining step number
self.stepCount += 1
res = application._analyseJob(self)
if not res['OK']:
return res
res = application._checkWorkflowConsistency()
if not res['OK']:
self.log.error("%s failed to check its consistency:" % application, "%s" % res['Message'])
return S_ERROR("%s failed to check its consistency: %s" % (application, res['Message']))
##Now we can create the step and add it to the workflow
#First we need a unique name, let's use the application name and step number
stepname = "%s_step_%s" % (application.appname, self.stepCount)
stepdefinition = StepDefinition(stepname)
self.steps.append(stepdefinition)
##Set the modules needed by the application
res = self._jobSpecificModules(application, stepdefinition)
if not res['OK']:
self.log.error("Failed to add modules:", "%s" % res['Message'])
return S_ERROR("Failed to add modules: %s" % res['Message'])
### add the parameters to the step
res = application._addParametersToStep(stepdefinition)
if not res['OK']:
self.log.error("Failed to add parameters:", "%s" % res['Message'])
return S_ERROR("Failed to add parameters: %s" % res['Message'])
##Now the step is defined, let's add it to the workflow
self.workflow.addStep(stepdefinition)
###Now we need to get a step instance object to set the parameters' values
stepInstance = self.workflow.createStepInstance(stepdefinition.getType(), stepname)
##Set the parameters values to the step instance
res = application._setStepParametersValues(stepInstance)
if not res['OK']:
self.log.error("Failed to resolve parameters values:", "%s" % res['Message'])
return S_ERROR("Failed to resolve parameters values: %s" % res['Message'])
res = application._resolveLinkedStepParameters(stepInstance)
if not res['OK']:
self.log.error("Failed to resolve linked parameters:", "%s" % res['Message'])
return S_ERROR("Failed to resolve linked parameters: %s" % res['Message'])
#Now prevent overwriting of parameter values.
application._addedtojob()
self._addParameter(self.workflow, 'TotalSteps', 'String', self.stepCount, 'Total number of steps')
return S_OK()
def getStepDefinition(stepName,
modulesNameList=None,
importLine="""""",
parametersList=None):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition(moduleName)
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (importLine, moduleName), globals(),
locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(importLine, moduleName, moduleName))
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (alternativeImportLine, moduleName),
globals(), locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(alternativeImportLine, moduleName, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def addWrapper(self, logFile=''):
""" Overload the DIRAC.Job.setExecutable
"""
logFile = str(logFile)
stepDefn = 'WrapperStep'
stepName = 'RunWrapperStep'
moduleName = 'GlastWrapperCall'
module = ModuleDefinition(moduleName)
module.setDescription('The utility that calls the pipeline_wrapper.')
body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
module.setBody(body)
# Create Step definition
step = StepDefinition(stepDefn)
step.addModule(module)
moduleInstance = step.createModuleInstance('GlastWrapperCall',
stepDefn)
# Define step parameters
step.addParameter(
Parameter("logFile", "", "string", "", "", False, False,
'Log file name'))
self.addToOutputSandbox.append(logFile)
self.workflow.addStep(step)
# Define Step and its variables
stepInstance = self.workflow.createStepInstance(stepDefn, stepName)
stepInstance.setValue("logFile", logFile)
return S_OK()
def createSingleModuleWorkflow(module, name):
''' Creates a workflow based on a single module definition
'''
moduleType = module.getType()
moduleName = name
workflow = Workflow()
step = StepDefinition(moduleType + '_step')
step.addModule(module)
moduleInstance = step.createModuleInstance(moduleType, moduleName)
step.addParameter(moduleInstance.parameters.getInput())
workflow.addParameter(moduleInstance.parameters.getInput())
workflow.addStep(step)
stepInstance = workflow.createStepInstance(moduleType + '_step',
moduleName + '_step')
# Propagate the module input parameters to the workflow level
moduleInstance.linkParameterUp(moduleInstance.parameters.getInput())
stepInstance.linkParameterUp(moduleInstance.parameters.getInput())
workflow.setName(name)
workflow.setDescription('Single module workflow from ' + moduleType +
' type module')
workflow.setDescrShort(moduleType + ' workflow')
return workflow
def createSingleModuleWorkflow(module,name):
''' Creates a workflow based on a single module definition
'''
moduleType = module.getType()
moduleName = name
workflow = Workflow()
step = StepDefinition(moduleType+'_step')
step.addModule(module)
moduleInstance = step.createModuleInstance(moduleType,moduleName)
step.addParameter(moduleInstance.parameters.getInput())
workflow.addParameter(moduleInstance.parameters.getInput())
workflow.addStep(step)
stepInstance = workflow.createStepInstance(moduleType+'_step',moduleName+'_step')
# Propagate the module input parameters to the workflow level
moduleInstance.linkParameterUp(moduleInstance.parameters.getInput())
stepInstance.linkParameterUp(moduleInstance.parameters.getInput())
workflow.setName(name)
workflow.setDescription('Single module workflow from '+moduleType+' type module')
workflow.setDescrShort(moduleType+' workflow')
return workflow
def getStepDefinition(stepName,
modulesNameList=None,
importLine="",
parametersList=None):
"""Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
module = None
if importLine:
try:
module = importlib.import_module(importLine + "." + moduleName)
except ImportError:
pass
# In case the importLine is not set, this is looking for a DIRAC extension, if any
if module is None:
module = ObjectLoader().loadModule("Workflow.Modules." +
moduleName)["Value"]
# create the module definition
moduleDef = ModuleDefinition(moduleName)
moduleDef.setDescription(module.__doc__)
moduleDef.setBody("\nfrom %s import %s\n" %
(module.__name__, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def startElement(self, name, attrs):
# print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames()
self.clearCharacters(
) # clear to remove empty or nonprintable characters
if name == "Workflow":
if self.root == None: # if root not defined by constractor
self.root = Workflow()
self.stack.append(self.root)
elif name == "StepDefinition":
obj = StepDefinition("TemporaryXMLObject_StepDefinition")
if self.root == None: # in case we are saving Step only
self.root = obj
self.stack.append(obj)
elif name == "StepInstance":
obj = StepInstance("TemporaryXMLObject_StepInstance")
self.stack.append(obj)
elif name == "ModuleDefinition":
obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition")
if self.root == None: # in case we are saving Module only
self.root = obj
self.stack.append(obj)
elif name == "ModuleInstance":
obj = ModuleInstance("TemporaryXMLObject_ModuleInstance")
self.stack.append(obj)
elif name == "Parameter":
obj = Parameter(str(attrs['name']), None, str(attrs['type']),
str(attrs['linked_module']),
str(attrs['linked_parameter']), str(attrs['in']),
str(attrs['out']), str(attrs['description']))
self.stack.append(obj)
# TEMPORARY CODE
elif name == "origin" or name == "version" or name == "name" or name == "type" or name == "value" or\
name == "required" or name == "descr_short" or name == "name" or name == "type" or name == "description" or name == "body":
pass
else:
print "UNTREATED! startElement name=", name, "attr=", attrs.getLength(
), attrs.getNames()
pass
def getStepDefinition( stepName, modulesNameList = [], importLine = """""", parametersList = [] ):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition( stepName )
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition( moduleName )
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription( getattr( __import__( "%s.%s" % ( importLine, moduleName ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( importLine, moduleName, moduleName ) )
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription( getattr( __import__( "%s.%s" % ( alternativeImportLine, moduleName ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( alternativeImportLine, moduleName, moduleName ) )
# add the module to the step, and instance it
stepDef.addModule( moduleDef )
stepDef.createModuleInstance( module_type = moduleName, name = moduleName )
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter( pName, pValue, pType, "", "", True, False, pDesc )
stepDef.addParameter( Parameter( parameter = p ) )
return stepDef
def _addRealFinalization(self):
""" See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest, 'enable', 'bool', False,
'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('ILDRegisterOutputData')
registerdata.setDescription(
'Module to add in the metadata catalog the relevant info about the files'
)
self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'ILDRegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'UploadLogFile')
logUpload.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData',
'dataUpload')
up.setValue("enable", self.finalsdict['uploadData'])
finalization.addModule(registerdata)
# TODO: create ILDRegisterOutputData
ro = finalization.createModuleInstance('ILDRegisterOutputData',
'ILDRegisterOutputData')
ro.setValue("enable", self.finalsdict['registerData'])
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
ul.setValue("enable", self.finalsdict['uploadLog'])
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest',
'failoverRequest')
fr.setValue("enable", self.finalsdict['sendFailover'])
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
def test__getStepDefinition( self ):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition( 'Script' )
body = importLine.replace( '<MODULE>', 'Script' )
gaudiApp.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'Script' ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
gaudiApp.setBody( body )
genBKReport = ModuleDefinition( 'FailoverRequest' )
body = importLine.replace( '<MODULE>', 'FailoverRequest' )
genBKReport.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'FailoverRequest' ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
genBKReport.setBody( body )
# step
appDefn = StepDefinition( 'App_Step' )
appDefn.addModule( gaudiApp )
appDefn.createModuleInstance( 'Script', 'Script' )
appDefn.addModule( genBKReport )
appDefn.createModuleInstance( 'FailoverRequest', 'FailoverRequest' )
appDefn.addParameterLinked( gaudiApp.parameters )
stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'] )
self.assertTrue( str( appDefn ) == str( stepDef ) )
self.job._addParameter( appDefn, 'name', 'type', 'value', 'desc' )
self.job._addParameter( appDefn, 'name1', 'type1', 'value1', 'desc1' )
stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'],
parametersList = [[ 'name', 'type', 'value', 'desc' ],
[ 'name1', 'type1', 'value1', 'desc1' ]] )
self.assertTrue( str( appDefn ) == str( stepDef ) )
def _addRealFinalization(self):
""" This is called at creation: now that the workflow is created at the last minute,
we need to add this also at the last minute
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest, 'enable', 'bool', False,
'EnableFlag')
body = importLine.replace('<MODULE>', 'FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('RegisterOutputData')
registerdata.setDescription(
'Module to add in the metadata catalog the relevant info about the files'
)
self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'RegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadLogFile')
logUpload.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData',
'dataUpload')
up.setValue("enable", self.finalsdict['uploadData'])
finalization.addModule(registerdata)
ro = finalization.createModuleInstance('RegisterOutputData',
'RegisterOutputData')
ro.setValue("enable", self.finalsdict['registerData'])
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
ul.setValue("enable", self.finalsdict['uploadLog'])
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest',
'failoverRequest')
fr.setValue("enable", self.finalsdict['sendFailover'])
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step
def test__getStepDefinition(self):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition("Script")
body = importLine.replace("<MODULE>", "Script")
gaudiApp.setDescription(
getattr(
__import__("%s.%s" % ("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), "__doc__"
)
)
gaudiApp.setBody(body)
genBKReport = ModuleDefinition("FailoverRequest")
body = importLine.replace("<MODULE>", "FailoverRequest")
genBKReport.setDescription(
getattr(
__import__("%s.%s" % ("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"]),
"__doc__",
)
)
genBKReport.setBody(body)
# step
appDefn = StepDefinition("App_Step")
appDefn.addModule(gaudiApp)
appDefn.createModuleInstance("Script", "Script")
appDefn.addModule(genBKReport)
appDefn.createModuleInstance("FailoverRequest", "FailoverRequest")
appDefn.addParameterLinked(gaudiApp.parameters)
stepDef = getStepDefinition("App_Step", ["Script", "FailoverRequest"])
self.assertTrue(str(appDefn) == str(stepDef))
self.job._addParameter(appDefn, "name", "type", "value", "desc")
self.job._addParameter(appDefn, "name1", "type1", "value1", "desc1")
stepDef = getStepDefinition(
"App_Step",
["Script", "FailoverRequest"],
parametersList=[["name", "type", "value", "desc"], ["name1", "type1", "value1", "desc1"]],
)
self.assertTrue(str(appDefn) == str(stepDef))
def append(self,application):
""" Helper function
This is the main part: call for every application
@param application: Application instance
"""
#Start by defining step number
self.stepnumber = len(self.steps) + 1
res = application._analyseJob(self)
if not res['OK']:
return res
res = application._checkConsistency()
if not res['OK']:
self.log.error("%s failed to check its consistency: %s"%(application,res['Message']))
return S_ERROR("%s failed to check its consistency: %s"%(application,res['Message']))
res = self._jobSpecificParams(application)
if not res['OK']:
self.log.error("%s failed job specific checks: %s"%(application,res['Message']))
return S_ERROR("%s failed job specific checks: %s"%(application,res['Message']))
res = application._checkFinalConsistency()
if not res['OK']:
self.log.error("%s failed to check its consistency: %s"%(application,res['Message']))
return S_ERROR("%s failed to check its consistency: %s"%(application,res['Message']))
### Once the consistency has been checked, we can add the application to the list of apps.
self.applicationlist.append(application)
##Get the application's sandbox and add it to the job's
for isb in application.inputSB:
if not isb in self.inputsandbox:
self.inputsandbox.append(isb)
#self.inputsandbox.extend(application.inputSB)
##Now we can create the step and add it to the workflow
#First we need a unique name, let's use the application name and step number
stepname = "%s_step_%s"%(application.appname,self.stepnumber)
stepdefinition = StepDefinition(stepname)
self.steps.append(stepdefinition)
##Set the modules needed by the application
res = self._jobSpecificModules(application,stepdefinition)
if not res['OK']:
self.log.error("Failed to add modules: %s"%res['Message'])
return S_ERROR("Failed to add modules: %s"%res['Message'])
### add the parameters to the step
res = application._addParametersToStep(stepdefinition)
if not res['OK']:
self.log.error("Failed to add parameters: %s"%res['Message'])
return S_ERROR("Failed to add parameters: %s"%res['Message'])
##Now the step is defined, let's add it to the workflow
self.workflow.addStep(stepdefinition)
###Now we need to get a step instance object to set the parameters' values
stepInstance = self.workflow.createStepInstance(stepdefinition.getType(),stepname)
##Set the parameters values to the step instance
res = application._setStepParametersValues(stepInstance)
if not res['OK']:
self.log.error("Failed to resolve parameters values: %s"%res['Message'])
return S_ERROR("Failed to resolve parameters values: %s"%res['Message'])
##stepInstance.setLink("InputFile",here lies the step name of the linked step, maybe get it from the application,"OutputFile")
res = application._resolveLinkedStepParameters(stepInstance)
if not res['OK']:
self.log.error("Failed to resolve linked parameters: %s"%res['Message'])
return S_ERROR("Failed to resolve linked parameters: %s"%res['Message'])
#Now prevent overwriting of parameter values.
application._addedtojob()
self._addParameter(self.workflow, 'TotalSteps', 'String', self.stepnumber, 'Total number of steps')
if application.nbevts:
self._addParameter(self.workflow, 'NbOfEvts', 'int', application.nbevts, "Number of events to process")
##Finally, add the software packages if needed
if application.appname and application.version:
self._addSoftware(application.appname, application.version)
return S_OK()
def test__getStepDefinition(self):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition('Script')
body = importLine.replace('<MODULE>', 'Script')
gaudiApp.setDescription(getattr(__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'Script'),
globals(), locals(), ['__doc__']),
"__doc__"))
gaudiApp.setBody(body)
genBKReport = ModuleDefinition('FailoverRequest')
body = importLine.replace('<MODULE>', 'FailoverRequest')
genBKReport.setDescription(getattr(__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'FailoverRequest'),
globals(), locals(), ['__doc__']),
"__doc__"))
genBKReport.setBody(body)
# step
appDefn = StepDefinition('App_Step')
appDefn.addModule(gaudiApp)
appDefn.createModuleInstance('Script', 'Script')
appDefn.addModule(genBKReport)
appDefn.createModuleInstance('FailoverRequest', 'FailoverRequest')
appDefn.addParameterLinked(gaudiApp.parameters)
stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'])
self.assertTrue(str(appDefn) == str(stepDef))
self.job._addParameter(appDefn, 'name', 'type', 'value', 'desc')
self.job._addParameter(appDefn, 'name1', 'type1', 'value1', 'desc1')
stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'],
parametersList=[['name', 'type', 'value', 'desc'],
['name1', 'type1', 'value1', 'desc1']])
self.assertTrue(str(appDefn) == str(stepDef))
def _addToWorkflow(self):
""" This is called just before submission. It creates the actual workflow.
The linking of parameters can only be done here
"""
for application in self.applicationlist:
#Start by defining step number
self.stepCount += 1
res = application._analyseJob(self)
if not res['OK']:
return res
res = application._checkWorkflowConsistency()
if not res['OK']:
self.log.error(
"%s failed to check its consistency:" % application,
"%s" % res['Message'])
return S_ERROR("%s failed to check its consistency: %s" %
(application, res['Message']))
##Now we can create the step and add it to the workflow
#First we need a unique name, let's use the application name and step number
stepname = "%s_step_%s" % (application.appname, self.stepCount)
stepdefinition = StepDefinition(stepname)
self.steps.append(stepdefinition)
##Set the modules needed by the application
res = self._jobSpecificModules(application, stepdefinition)
if not res['OK']:
self.log.error("Failed to add modules:", "%s" % res['Message'])
return S_ERROR("Failed to add modules: %s" % res['Message'])
### add the parameters to the step
res = application._addParametersToStep(stepdefinition)
if not res['OK']:
self.log.error("Failed to add parameters:",
"%s" % res['Message'])
return S_ERROR("Failed to add parameters: %s" % res['Message'])
##Now the step is defined, let's add it to the workflow
self.workflow.addStep(stepdefinition)
###Now we need to get a step instance object to set the parameters' values
stepInstance = self.workflow.createStepInstance(
stepdefinition.getType(), stepname)
##Set the parameters values to the step instance
res = application._setStepParametersValues(stepInstance)
if not res['OK']:
self.log.error("Failed to resolve parameters values:",
"%s" % res['Message'])
return S_ERROR("Failed to resolve parameters values: %s" %
res['Message'])
res = application._resolveLinkedStepParameters(stepInstance)
if not res['OK']:
self.log.error("Failed to resolve linked parameters:",
"%s" % res['Message'])
return S_ERROR("Failed to resolve linked parameters: %s" %
res['Message'])
#Now prevent overwriting of parameter values.
application._addedtojob()
self._addParameter(self.workflow, 'TotalSteps', 'String',
self.stepCount, 'Total number of steps')
return S_OK()
def _addRealFinalization( self ):
""" See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition( 'UploadOutputData' )
dataUpload.setDescription( 'Uploads the output data' )
self._addParameter( dataUpload, 'enable', 'bool', False, 'EnableFlag' )
body = string.replace( importLine, '<MODULE>', 'UploadOutputData' )
dataUpload.setBody( body )
failoverRequest = ModuleDefinition( 'FailoverRequest' )
failoverRequest.setDescription( 'Sends any failover requests' )
self._addParameter( failoverRequest, 'enable', 'bool', False, 'EnableFlag' )
body = string.replace( importLine, '<MODULE>', 'FailoverRequest' )
failoverRequest.setBody( body )
registerdata = ModuleDefinition( 'ILDRegisterOutputData' )
registerdata.setDescription( 'Module to add in the metadata catalog the relevant info about the files' )
self._addParameter( registerdata, 'enable', 'bool', False, 'EnableFlag' )
body = string.replace( importLine, '<MODULE>', 'ILDRegisterOutputData' )
registerdata.setBody( body )
logUpload = ModuleDefinition( 'UploadLogFile' )
logUpload.setDescription( 'Uploads the output log files' )
self._addParameter( logUpload, 'enable', 'bool', False, 'EnableFlag' )
body = string.replace( importLine, '<MODULE>', 'UploadLogFile' )
logUpload.setBody( body )
finalization = StepDefinition( 'Job_Finalization' )
finalization.addModule( dataUpload )
up = finalization.createModuleInstance( 'UploadOutputData', 'dataUpload' )
up.setValue( "enable", self.finalsdict['uploadData'] )
finalization.addModule( registerdata )
# TODO: create ILDRegisterOutputData
ro = finalization.createModuleInstance( 'ILDRegisterOutputData', 'ILDRegisterOutputData' )
ro.setValue( "enable", self.finalsdict['registerData'] )
finalization.addModule( logUpload )
ul = finalization.createModuleInstance( 'UploadLogFile', 'logUpload' )
ul.setValue( "enable", self.finalsdict['uploadLog'] )
finalization.addModule( failoverRequest )
fr = finalization.createModuleInstance( 'FailoverRequest', 'failoverRequest' )
fr.setValue( "enable", self.finalsdict['sendFailover'] )
self.workflow.addStep( finalization )
self.workflow.createStepInstance( 'Job_Finalization', 'finalization' )
return S_OK()
def addFinalization(self, uploadData=False, registerData=False, uploadLog = False, sendFailover=False):
""" Add finalization step
@param uploadData: Upload or not the data to the storage
@param uploadLog: Upload log file to storage (currently only available for admins, thus add them to OutputSandbox)
@param sendFailover: Send Failover requests, and declare files as processed or unused in transfDB
@param registerData: Register data in the file catalog
@todo: Do the registration only once, instead of once for each job
"""
self.importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload,'enable','bool',False,'EnableFlag')
body = string.replace(self.importLine,'<MODULE>','UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest,'enable','bool',False,'EnableFlag')
body = string.replace(self.importLine,'<MODULE>','FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('DBDGenRegisterOutputData')
registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files')
self._addParameter(registerdata,'enable','bool',False,'EnableFlag')
body = string.replace(self.importLine,'<MODULE>','DBDGenRegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload,'enable','bool',False,'EnableFlag')
body = string.replace(self.importLine,'<MODULE>','UploadLogFile')
logUpload.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData','dataUpload')
up.setValue("enable",uploadData)
finalization.addModule(registerdata)
ro = finalization.createModuleInstance('DBDGenRegisterOutputData','DBDGenRegisterOutputData')
ro.setValue("enable",registerData)
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile','logUpload')
ul.setValue("enable",uploadLog)
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest','failoverRequest')
fr.setValue("enable",sendFailover)
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step
def _addRealFinalization(self):
""" This is called at creation: now that the workflow is created at the last minute,
we need to add this also at the last minute
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('RegisterOutputData')
registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files')
self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'RegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadLogFile')
logUpload.setBody(body)
errorReport = ModuleDefinition('ReportErrors')
errorReport.setDescription('Reports errors at the end')
body = importLine.replace('<MODULE>', 'ReportErrors')
errorReport.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData', 'dataUpload')
up.setValue("enable", self.finalsdict['uploadData'])
finalization.addModule(registerdata)
ro = finalization.createModuleInstance('RegisterOutputData', 'RegisterOutputData')
ro.setValue("enable", self.finalsdict['registerData'])
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
ul.setValue("enable", self.finalsdict['uploadLog'])
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest')
fr.setValue("enable", self.finalsdict['sendFailover'])
finalization.addModule(errorReport)
fr = finalization.createModuleInstance('ReportErrors', 'reportErrors')
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()