def addWrapper(self, logFile = ''):
""" Overload the DIRAC.Job.setExecutable
"""
logFile = str(logFile)
stepDefn = 'WrapperStep'
stepName = 'RunWrapperStep'
moduleName = 'GlastWrapperCall'
module = ModuleDefinition( moduleName )
module.setDescription( 'The utility that calls the pipeline_wrapper.' )
body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
module.setBody( body )
# Create Step definition
step = StepDefinition( stepDefn )
step.addModule( module )
moduleInstance = step.createModuleInstance( 'GlastWrapperCall', stepDefn )
# Define step parameters
step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
self.addToOutputSandbox.append( logFile )
self.workflow.addStep( step )
# Define Step and its variables
stepInstance = self.workflow.createStepInstance( stepDefn, stepName )
stepInstance.setValue( "logFile", logFile )
return S_OK()
def _createModuleDefinition(self): """ Create Module definition. As it's generic code, all apps will use this. """ moduledefinition = ModuleDefinition(self._modulename) moduledefinition.setDescription(self._moduledescription) body = 'from %s.%s import %s\n' % (self._importLocation, self._modulename, self._modulename) moduledefinition.setBody(body) return moduledefinition
def _getComputeOutputDataListModule(self):
""" This is separated from the applications as this is used in production jobs only.
"""
moduledefinition = ModuleDefinition("ComputeOutputDataList")
moduledefinition.setDescription("Compute the output data list to be treated by the last finalization")
body = 'from %s.%s import %s\n' % (self._importLocation, "ComputeOutputDataList", "ComputeOutputDataList" )
moduledefinition.setBody(body)
return moduledefinition
def getStepDefinition(stepName,
modulesNameList=None,
importLine="""""",
parametersList=None):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition(moduleName)
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (importLine, moduleName), globals(),
locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(importLine, moduleName, moduleName))
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription(
getattr(
__import__("%s.%s" % (alternativeImportLine, moduleName),
globals(), locals(), ['__doc__']), "__doc__"))
moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
(alternativeImportLine, moduleName, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def _getUserOutputDataModule(self):
""" This is separated as not all applications require user specific output data (i.e. GetSRMFile and Overlay). Only used in UserJobs.
The UserJobFinalization only runs last. It's called every step, but is running only if last.
"""
moduledefinition = ModuleDefinition('UserJobFinalization')
moduledefinition.setDescription('Uploads user output data files with specific policies.')
body = 'from %s.%s import %s\n' % (self._importLocation, 'UserJobFinalization', 'UserJobFinalization')
moduledefinition.setBody(body)
return moduledefinition
def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step
def test__getStepDefinition( self ):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition( 'Script' )
body = importLine.replace( '<MODULE>', 'Script' )
gaudiApp.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'Script' ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
gaudiApp.setBody( body )
genBKReport = ModuleDefinition( 'FailoverRequest' )
body = importLine.replace( '<MODULE>', 'FailoverRequest' )
genBKReport.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'FailoverRequest' ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
genBKReport.setBody( body )
# step
appDefn = StepDefinition( 'App_Step' )
appDefn.addModule( gaudiApp )
appDefn.createModuleInstance( 'Script', 'Script' )
appDefn.addModule( genBKReport )
appDefn.createModuleInstance( 'FailoverRequest', 'FailoverRequest' )
appDefn.addParameterLinked( gaudiApp.parameters )
stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'] )
self.assertTrue( str( appDefn ) == str( stepDef ) )
self.job._addParameter( appDefn, 'name', 'type', 'value', 'desc' )
self.job._addParameter( appDefn, 'name1', 'type1', 'value1', 'desc1' )
stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'],
parametersList = [[ 'name', 'type', 'value', 'desc' ],
[ 'name1', 'type1', 'value1', 'desc1' ]] )
self.assertTrue( str( appDefn ) == str( stepDef ) )
def addWrapper(self, logFile=''):
""" Overload the DIRAC.Job.setExecutable
"""
logFile = str(logFile)
stepDefn = 'WrapperStep'
stepName = 'RunWrapperStep'
moduleName = 'GlastWrapperCall'
module = ModuleDefinition(moduleName)
module.setDescription('The utility that calls the pipeline_wrapper.')
body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
module.setBody(body)
# Create Step definition
step = StepDefinition(stepDefn)
step.addModule(module)
moduleInstance = step.createModuleInstance('GlastWrapperCall',
stepDefn)
# Define step parameters
step.addParameter(
Parameter("logFile", "", "string", "", "", False, False,
'Log file name'))
self.addToOutputSandbox.append(logFile)
self.workflow.addStep(step)
# Define Step and its variables
stepInstance = self.workflow.createStepInstance(stepDefn, stepName)
stepInstance.setValue("logFile", logFile)
return S_OK()
def _createModuleDefinition(self):
""" Create Module definition. As it's generic code, all apps will use this.
"""
moduledefinition = ModuleDefinition(self._modulename)
moduledefinition.setDescription(self._moduledescription)
body = 'from %s.%s import %s\n' % (self._importLocation, self._modulename, self._modulename)
moduledefinition.setBody(body)
return moduledefinition
def _getComputeOutputDataListModule(self):
""" This is separated from the applications as this is used in production jobs only.
"""
moduledefinition = ModuleDefinition("ComputeOutputDataList")
moduledefinition.setDescription("Compute the output data list to be treated by the last finalization")
body = 'from %s.%s import %s\n' % (self._importLocation, "ComputeOutputDataList", "ComputeOutputDataList" )
moduledefinition.setBody(body)
return moduledefinition
def getStepDefinition( stepName, modulesNameList = [], importLine = """""", parametersList = [] ):
""" Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
# In case the importLine is not set, this is looking for a DIRAC extension, if any.
# The extension is supposed to be called ExtDIRAC.
if not importLine:
importLine = "DIRAC.Workflow.Modules"
for ext in getCSExtensions():
if ext.lower() == getVO():
importLine = ext + "DIRAC.Workflow.Modules"
break
stepDef = StepDefinition( stepName )
for moduleName in modulesNameList:
# create the module definition
moduleDef = ModuleDefinition( moduleName )
try:
# Look in the importLine given, or the DIRAC if the given location can't be imported
moduleDef.setDescription( getattr( __import__( "%s.%s" % ( importLine, moduleName ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( importLine, moduleName, moduleName ) )
except ImportError:
alternativeImportLine = "DIRAC.Workflow.Modules"
moduleDef.setDescription( getattr( __import__( "%s.%s" % ( alternativeImportLine, moduleName ),
globals(), locals(), ['__doc__'] ),
"__doc__" ) )
moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( alternativeImportLine, moduleName, moduleName ) )
# add the module to the step, and instance it
stepDef.addModule( moduleDef )
stepDef.createModuleInstance( module_type = moduleName, name = moduleName )
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter( pName, pValue, pType, "", "", True, False, pDesc )
stepDef.addParameter( Parameter( parameter = p ) )
return stepDef
def _getUserOutputDataModule(self):
""" This is separated as not all applications require user specific output data (i.e. GetSRMFile and Overlay).
Only used in UserJobs.
The UserJobFinalization only runs last. It's called every step, but is running only if last.
"""
moduledefinition = ModuleDefinition('UserJobFinalization')
moduledefinition.setDescription('Uploads user output data files with specific policies.')
body = 'from %s.%s import %s\n' % (self._importLocation, 'UserJobFinalization', 'UserJobFinalization')
moduledefinition.setBody(body)
return moduledefinition
def startElement(self, name, attrs):
# print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames()
self.clearCharacters(
) # clear to remove empty or nonprintable characters
if name == "Workflow":
if self.root == None: # if root not defined by constractor
self.root = Workflow()
self.stack.append(self.root)
elif name == "StepDefinition":
obj = StepDefinition("TemporaryXMLObject_StepDefinition")
if self.root == None: # in case we are saving Step only
self.root = obj
self.stack.append(obj)
elif name == "StepInstance":
obj = StepInstance("TemporaryXMLObject_StepInstance")
self.stack.append(obj)
elif name == "ModuleDefinition":
obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition")
if self.root == None: # in case we are saving Module only
self.root = obj
self.stack.append(obj)
elif name == "ModuleInstance":
obj = ModuleInstance("TemporaryXMLObject_ModuleInstance")
self.stack.append(obj)
elif name == "Parameter":
obj = Parameter(str(attrs['name']), None, str(attrs['type']),
str(attrs['linked_module']),
str(attrs['linked_parameter']), str(attrs['in']),
str(attrs['out']), str(attrs['description']))
self.stack.append(obj)
# TEMPORARY CODE
elif name == "origin" or name == "version" or name == "name" or name == "type" or name == "value" or\
name == "required" or name == "descr_short" or name == "name" or name == "type" or name == "description" or name == "body":
pass
else:
print "UNTREATED! startElement name=", name, "attr=", attrs.getLength(
), attrs.getNames()
pass
def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step
def getStepDefinition(stepName,
modulesNameList=None,
importLine="",
parametersList=None):
"""Given a name, a list of modules name, and a list of parameters, returns a step definition.
Remember that Step definition = Parameters + Module Instances
"""
if modulesNameList is None:
modulesNameList = []
if parametersList is None:
parametersList = []
stepDef = StepDefinition(stepName)
for moduleName in modulesNameList:
module = None
if importLine:
try:
module = importlib.import_module(importLine + "." + moduleName)
except ImportError:
pass
# In case the importLine is not set, this is looking for a DIRAC extension, if any
if module is None:
module = ObjectLoader().loadModule("Workflow.Modules." +
moduleName)["Value"]
# create the module definition
moduleDef = ModuleDefinition(moduleName)
moduleDef.setDescription(module.__doc__)
moduleDef.setBody("\nfrom %s import %s\n" %
(module.__name__, moduleName))
# add the module to the step, and instance it
stepDef.addModule(moduleDef)
stepDef.createModuleInstance(module_type=moduleName, name=moduleName)
# add parameters to the module definition
for pName, pType, pValue, pDesc in parametersList:
p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
stepDef.addParameter(Parameter(parameter=p))
return stepDef
def addFinalization(self, uploadData=False, registerData=False, uploadLog = False, sendFailover=False):
""" Add finalization step
@param uploadData: Upload or not the data to the storage
@param uploadLog: Upload log file to storage (currently only available for admins, thus add them to OutputSandbox)
@param sendFailover: Send Failover requests, and declare files as processed or unused in transfDB
@param registerData: Register data in the file catalog
@todo: Do the registration only once, instead of once for each job
"""
self.importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload,'enable','bool',False,'EnableFlag')
body = string.replace(self.importLine,'<MODULE>','UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest,'enable','bool',False,'EnableFlag')
body = string.replace(self.importLine,'<MODULE>','FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('DBDGenRegisterOutputData')
registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files')
self._addParameter(registerdata,'enable','bool',False,'EnableFlag')
body = string.replace(self.importLine,'<MODULE>','DBDGenRegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload,'enable','bool',False,'EnableFlag')
body = string.replace(self.importLine,'<MODULE>','UploadLogFile')
logUpload.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData','dataUpload')
up.setValue("enable",uploadData)
finalization.addModule(registerdata)
ro = finalization.createModuleInstance('DBDGenRegisterOutputData','DBDGenRegisterOutputData')
ro.setValue("enable",registerData)
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile','logUpload')
ul.setValue("enable",uploadLog)
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest','failoverRequest')
fr.setValue("enable",sendFailover)
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
def _addRealFinalization( self ):
""" See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition( 'UploadOutputData' )
dataUpload.setDescription( 'Uploads the output data' )
self._addParameter( dataUpload, 'enable', 'bool', False, 'EnableFlag' )
body = string.replace( importLine, '<MODULE>', 'UploadOutputData' )
dataUpload.setBody( body )
failoverRequest = ModuleDefinition( 'FailoverRequest' )
failoverRequest.setDescription( 'Sends any failover requests' )
self._addParameter( failoverRequest, 'enable', 'bool', False, 'EnableFlag' )
body = string.replace( importLine, '<MODULE>', 'FailoverRequest' )
failoverRequest.setBody( body )
registerdata = ModuleDefinition( 'ILDRegisterOutputData' )
registerdata.setDescription( 'Module to add in the metadata catalog the relevant info about the files' )
self._addParameter( registerdata, 'enable', 'bool', False, 'EnableFlag' )
body = string.replace( importLine, '<MODULE>', 'ILDRegisterOutputData' )
registerdata.setBody( body )
logUpload = ModuleDefinition( 'UploadLogFile' )
logUpload.setDescription( 'Uploads the output log files' )
self._addParameter( logUpload, 'enable', 'bool', False, 'EnableFlag' )
body = string.replace( importLine, '<MODULE>', 'UploadLogFile' )
logUpload.setBody( body )
finalization = StepDefinition( 'Job_Finalization' )
finalization.addModule( dataUpload )
up = finalization.createModuleInstance( 'UploadOutputData', 'dataUpload' )
up.setValue( "enable", self.finalsdict['uploadData'] )
finalization.addModule( registerdata )
# TODO: create ILDRegisterOutputData
ro = finalization.createModuleInstance( 'ILDRegisterOutputData', 'ILDRegisterOutputData' )
ro.setValue( "enable", self.finalsdict['registerData'] )
finalization.addModule( logUpload )
ul = finalization.createModuleInstance( 'UploadLogFile', 'logUpload' )
ul.setValue( "enable", self.finalsdict['uploadLog'] )
finalization.addModule( failoverRequest )
fr = finalization.createModuleInstance( 'FailoverRequest', 'failoverRequest' )
fr.setValue( "enable", self.finalsdict['sendFailover'] )
self.workflow.addStep( finalization )
self.workflow.createStepInstance( 'Job_Finalization', 'finalization' )
return S_OK()
def _addRealFinalization(self):
""" This is called at creation: now that the workflow is created at the last minute,
we need to add this also at the last minute
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest, 'enable', 'bool', False,
'EnableFlag')
body = importLine.replace('<MODULE>', 'FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('RegisterOutputData')
registerdata.setDescription(
'Module to add in the metadata catalog the relevant info about the files'
)
self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'RegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadLogFile')
logUpload.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData',
'dataUpload')
up.setValue("enable", self.finalsdict['uploadData'])
finalization.addModule(registerdata)
ro = finalization.createModuleInstance('RegisterOutputData',
'RegisterOutputData')
ro.setValue("enable", self.finalsdict['registerData'])
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
ul.setValue("enable", self.finalsdict['uploadLog'])
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest',
'failoverRequest')
fr.setValue("enable", self.finalsdict['sendFailover'])
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
def test__getStepDefinition(self):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition("Script")
body = importLine.replace("<MODULE>", "Script")
gaudiApp.setDescription(
getattr(
__import__("%s.%s" % ("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), "__doc__"
)
)
gaudiApp.setBody(body)
genBKReport = ModuleDefinition("FailoverRequest")
body = importLine.replace("<MODULE>", "FailoverRequest")
genBKReport.setDescription(
getattr(
__import__("%s.%s" % ("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"]),
"__doc__",
)
)
genBKReport.setBody(body)
# step
appDefn = StepDefinition("App_Step")
appDefn.addModule(gaudiApp)
appDefn.createModuleInstance("Script", "Script")
appDefn.addModule(genBKReport)
appDefn.createModuleInstance("FailoverRequest", "FailoverRequest")
appDefn.addParameterLinked(gaudiApp.parameters)
stepDef = getStepDefinition("App_Step", ["Script", "FailoverRequest"])
self.assertTrue(str(appDefn) == str(stepDef))
self.job._addParameter(appDefn, "name", "type", "value", "desc")
self.job._addParameter(appDefn, "name1", "type1", "value1", "desc1")
stepDef = getStepDefinition(
"App_Step",
["Script", "FailoverRequest"],
parametersList=[["name", "type", "value", "desc"], ["name1", "type1", "value1", "desc1"]],
)
self.assertTrue(str(appDefn) == str(stepDef))
def test__getStepDefinition(self):
importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
# modules
gaudiApp = ModuleDefinition('Script')
body = importLine.replace('<MODULE>', 'Script')
gaudiApp.setDescription(getattr(__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'Script'),
globals(), locals(), ['__doc__']),
"__doc__"))
gaudiApp.setBody(body)
genBKReport = ModuleDefinition('FailoverRequest')
body = importLine.replace('<MODULE>', 'FailoverRequest')
genBKReport.setDescription(getattr(__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'FailoverRequest'),
globals(), locals(), ['__doc__']),
"__doc__"))
genBKReport.setBody(body)
# step
appDefn = StepDefinition('App_Step')
appDefn.addModule(gaudiApp)
appDefn.createModuleInstance('Script', 'Script')
appDefn.addModule(genBKReport)
appDefn.createModuleInstance('FailoverRequest', 'FailoverRequest')
appDefn.addParameterLinked(gaudiApp.parameters)
stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'])
self.assertTrue(str(appDefn) == str(stepDef))
self.job._addParameter(appDefn, 'name', 'type', 'value', 'desc')
self.job._addParameter(appDefn, 'name1', 'type1', 'value1', 'desc1')
stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'],
parametersList=[['name', 'type', 'value', 'desc'],
['name1', 'type1', 'value1', 'desc1']])
self.assertTrue(str(appDefn) == str(stepDef))
def _addRealFinalization(self):
""" This is called at creation: now that the workflow is created at the last minute,
we need to add this also at the last minute
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('RegisterOutputData')
registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files')
self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'RegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
body = importLine.replace('<MODULE>', 'UploadLogFile')
logUpload.setBody(body)
errorReport = ModuleDefinition('ReportErrors')
errorReport.setDescription('Reports errors at the end')
body = importLine.replace('<MODULE>', 'ReportErrors')
errorReport.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData', 'dataUpload')
up.setValue("enable", self.finalsdict['uploadData'])
finalization.addModule(registerdata)
ro = finalization.createModuleInstance('RegisterOutputData', 'RegisterOutputData')
ro.setValue("enable", self.finalsdict['registerData'])
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
ul.setValue("enable", self.finalsdict['uploadLog'])
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest')
fr.setValue("enable", self.finalsdict['sendFailover'])
finalization.addModule(errorReport)
fr = finalization.createModuleInstance('ReportErrors', 'reportErrors')
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
def _addRealFinalization(self):
""" See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
"""
importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
dataUpload = ModuleDefinition('UploadOutputData')
dataUpload.setDescription('Uploads the output data')
self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'UploadOutputData')
dataUpload.setBody(body)
failoverRequest = ModuleDefinition('FailoverRequest')
failoverRequest.setDescription('Sends any failover requests')
self._addParameter(failoverRequest, 'enable', 'bool', False,
'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'FailoverRequest')
failoverRequest.setBody(body)
registerdata = ModuleDefinition('ILDRegisterOutputData')
registerdata.setDescription(
'Module to add in the metadata catalog the relevant info about the files'
)
self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'ILDRegisterOutputData')
registerdata.setBody(body)
logUpload = ModuleDefinition('UploadLogFile')
logUpload.setDescription('Uploads the output log files')
self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
body = string.replace(importLine, '<MODULE>', 'UploadLogFile')
logUpload.setBody(body)
finalization = StepDefinition('Job_Finalization')
finalization.addModule(dataUpload)
up = finalization.createModuleInstance('UploadOutputData',
'dataUpload')
up.setValue("enable", self.finalsdict['uploadData'])
finalization.addModule(registerdata)
# TODO: create ILDRegisterOutputData
ro = finalization.createModuleInstance('ILDRegisterOutputData',
'ILDRegisterOutputData')
ro.setValue("enable", self.finalsdict['registerData'])
finalization.addModule(logUpload)
ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
ul.setValue("enable", self.finalsdict['uploadLog'])
finalization.addModule(failoverRequest)
fr = finalization.createModuleInstance('FailoverRequest',
'failoverRequest')
fr.setValue("enable", self.finalsdict['sendFailover'])
self.workflow.addStep(finalization)
self.workflow.createStepInstance('Job_Finalization', 'finalization')
return S_OK()
