示例#1
0
    def addWrapper(self, logFile = ''):
        """ Overload the DIRAC.Job.setExecutable
        """
        logFile = str(logFile)
        stepDefn = 'WrapperStep'
        stepName = 'RunWrapperStep'

        moduleName = 'GlastWrapperCall'
        module = ModuleDefinition( moduleName )
        module.setDescription( 'The utility that calls the pipeline_wrapper.' )
        body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
        module.setBody( body )
        # Create Step definition
        step = StepDefinition( stepDefn )
        step.addModule( module )
        moduleInstance = step.createModuleInstance( 'GlastWrapperCall', stepDefn )
        # Define step parameters
        step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
        self.addToOutputSandbox.append( logFile )
        self.workflow.addStep( step )

        # Define Step and its variables  
        stepInstance = self.workflow.createStepInstance( stepDefn, stepName )
        stepInstance.setValue( "logFile", logFile )

        return S_OK()
示例#2
0
    def addWrapper(self, logFile=''):
        """ Overload the DIRAC.Job.setExecutable
        """
        logFile = str(logFile)
        stepDefn = 'WrapperStep'
        stepName = 'RunWrapperStep'

        moduleName = 'GlastWrapperCall'
        module = ModuleDefinition(moduleName)
        module.setDescription('The utility that calls the pipeline_wrapper.')
        body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
        module.setBody(body)
        # Create Step definition
        step = StepDefinition(stepDefn)
        step.addModule(module)
        moduleInstance = step.createModuleInstance('GlastWrapperCall',
                                                   stepDefn)
        # Define step parameters
        step.addParameter(
            Parameter("logFile", "", "string", "", "", False, False,
                      'Log file name'))
        self.addToOutputSandbox.append(logFile)
        self.workflow.addStep(step)

        # Define Step and its variables
        stepInstance = self.workflow.createStepInstance(stepDefn, stepName)
        stepInstance.setValue("logFile", logFile)

        return S_OK()
示例#3
0
  def _addRealFinalization(self):  
    """ This is called at creation: now that the workflow is created at the last minute,
    we need to add this also at the last minute
    """
    importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
    
    dataUpload = ModuleDefinition('UploadOutputData')
    dataUpload.setDescription('Uploads the output data')
    self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
    body = importLine.replace('<MODULE>', 'UploadOutputData')
    dataUpload.setBody(body)

    failoverRequest = ModuleDefinition('FailoverRequest')
    failoverRequest.setDescription('Sends any failover requests')
    self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag')
    body = importLine.replace('<MODULE>', 'FailoverRequest')
    failoverRequest.setBody(body)

    registerdata = ModuleDefinition('RegisterOutputData')
    registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files')
    self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
    body = importLine.replace('<MODULE>', 'RegisterOutputData')
    registerdata.setBody(body)

    logUpload = ModuleDefinition('UploadLogFile')
    logUpload.setDescription('Uploads the output log files')
    self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
    body = importLine.replace('<MODULE>', 'UploadLogFile')
    logUpload.setBody(body)

    errorReport = ModuleDefinition('ReportErrors')
    errorReport.setDescription('Reports errors at the end')
    body = importLine.replace('<MODULE>', 'ReportErrors')
    errorReport.setBody(body)

    finalization = StepDefinition('Job_Finalization')
    finalization.addModule(dataUpload)
    up = finalization.createModuleInstance('UploadOutputData', 'dataUpload')
    up.setValue("enable", self.finalsdict['uploadData'])

    finalization.addModule(registerdata)
    ro = finalization.createModuleInstance('RegisterOutputData', 'RegisterOutputData')
    ro.setValue("enable", self.finalsdict['registerData'])

    finalization.addModule(logUpload)
    ul  = finalization.createModuleInstance('UploadLogFile', 'logUpload')
    ul.setValue("enable", self.finalsdict['uploadLog'])

    finalization.addModule(failoverRequest)
    fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest')
    fr.setValue("enable", self.finalsdict['sendFailover'])

    finalization.addModule(errorReport)
    fr = finalization.createModuleInstance('ReportErrors', 'reportErrors')

    self.workflow.addStep(finalization)
    self.workflow.createStepInstance('Job_Finalization', 'finalization')

    return S_OK()
示例#4
0
 def _getComputeOutputDataListModule(self):
   """ This is separated from the applications as this is used in production jobs only.
   """
   moduledefinition = ModuleDefinition("ComputeOutputDataList")
   moduledefinition.setDescription("Compute the output data list to be treated by the last finalization")
   body = 'from %s.%s import %s\n' % (self._importLocation, "ComputeOutputDataList", "ComputeOutputDataList" )
   moduledefinition.setBody(body)
   return moduledefinition
示例#5
0
 def _getComputeOutputDataListModule(self):
     """ This is separated from the applications as this is used in production jobs only.
     """
     moduledefinition = ModuleDefinition("ComputeOutputDataList")
     moduledefinition.setDescription("Compute the output data list to be treated by the last finalization")
     body = 'from %s.%s import %s\n' % (self._importLocation, "ComputeOutputDataList", "ComputeOutputDataList" )
     moduledefinition.setBody(body)
     return moduledefinition
示例#6
0
 def _createModuleDefinition(self):
   """ Create Module definition. As it's generic code, all apps will use this.
   """
   moduledefinition = ModuleDefinition(self._modulename)
   moduledefinition.setDescription(self._moduledescription)
   body = 'from %s.%s import %s\n' % (self._importLocation, self._modulename, self._modulename)
   moduledefinition.setBody(body)
   return moduledefinition
示例#7
0
 def _createModuleDefinition(self):
     """ Create Module definition. As it's generic code, all apps will use this.
     """
     moduledefinition = ModuleDefinition(self._modulename)
     moduledefinition.setDescription(self._moduledescription)
     body = 'from %s.%s import %s\n' % (self._importLocation, self._modulename, self._modulename)
     moduledefinition.setBody(body)
     return moduledefinition
    def _addRealFinalization( self ):
        """ See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
        """
        importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
        dataUpload = ModuleDefinition( 'UploadOutputData' )
        dataUpload.setDescription( 'Uploads the output data' )
        self._addParameter( dataUpload, 'enable', 'bool', False, 'EnableFlag' )
        body = string.replace( importLine, '<MODULE>', 'UploadOutputData' )
        dataUpload.setBody( body )

        failoverRequest = ModuleDefinition( 'FailoverRequest' )
        failoverRequest.setDescription( 'Sends any failover requests' )
        self._addParameter( failoverRequest, 'enable', 'bool', False, 'EnableFlag' )
        body = string.replace( importLine, '<MODULE>', 'FailoverRequest' )
        failoverRequest.setBody( body )

        registerdata = ModuleDefinition( 'ILDRegisterOutputData' )
        registerdata.setDescription( 'Module to add in the metadata catalog the relevant info about the files' )
        self._addParameter( registerdata, 'enable', 'bool', False, 'EnableFlag' )
        body = string.replace( importLine, '<MODULE>', 'ILDRegisterOutputData' )
        registerdata.setBody( body )

        logUpload = ModuleDefinition( 'UploadLogFile' )
        logUpload.setDescription( 'Uploads the output log files' )
        self._addParameter( logUpload, 'enable', 'bool', False, 'EnableFlag' )
        body = string.replace( importLine, '<MODULE>', 'UploadLogFile' )
        logUpload.setBody( body )

        errorReport = ModuleDefinition('ReportErrors')
        errorReport.setDescription('Reports errors at the end')
        body = importLine.replace('<MODULE>', 'ReportErrors')
        errorReport.setBody(body)

        finalization = StepDefinition( 'Job_Finalization' )
        finalization.addModule( dataUpload )
        up = finalization.createModuleInstance( 'UploadOutputData', 'dataUpload' )
        up.setValue( "enable", self.finalsdict['uploadData'] )

        finalization.addModule( registerdata )
        # TODO: create ILDRegisterOutputData
        ro = finalization.createModuleInstance( 'ILDRegisterOutputData', 'ILDRegisterOutputData' )
        ro.setValue( "enable", self.finalsdict['registerData'] )

        finalization.addModule( logUpload )
        ul = finalization.createModuleInstance( 'UploadLogFile', 'logUpload' )
        ul.setValue( "enable", self.finalsdict['uploadLog'] )

        finalization.addModule( failoverRequest )
        fr = finalization.createModuleInstance( 'FailoverRequest', 'failoverRequest' )
        fr.setValue( "enable", self.finalsdict['sendFailover'] )

        finalization.addModule(errorReport)
        fr = finalization.createModuleInstance('ReportErrors', 'reportErrors')

        self.workflow.addStep( finalization )
        self.workflow.createStepInstance( 'Job_Finalization', 'finalization' )

        return S_OK() 
示例#9
0
def getStepDefinition(stepName,
                      modulesNameList=None,
                      importLine="""""",
                      parametersList=None):
    """ Given a name, a list of modules name, and a list of parameters, returns a step definition.
      Remember that Step definition = Parameters + Module Instances
  """

    if modulesNameList is None:
        modulesNameList = []

    if parametersList is None:
        parametersList = []

    # In case the importLine is not set, this is looking for a DIRAC extension, if any.
    # The extension is supposed to be called ExtDIRAC.
    if not importLine:
        importLine = "DIRAC.Workflow.Modules"
        for ext in getCSExtensions():
            if ext.lower() == getVO():
                importLine = ext + "DIRAC.Workflow.Modules"
                break

    stepDef = StepDefinition(stepName)

    for moduleName in modulesNameList:

        # create the module definition
        moduleDef = ModuleDefinition(moduleName)
        try:
            # Look in the importLine given, or the DIRAC if the given location can't be imported
            moduleDef.setDescription(
                getattr(
                    __import__("%s.%s" % (importLine, moduleName), globals(),
                               locals(), ['__doc__']), "__doc__"))
            moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
                              (importLine, moduleName, moduleName))
        except ImportError:
            alternativeImportLine = "DIRAC.Workflow.Modules"
            moduleDef.setDescription(
                getattr(
                    __import__("%s.%s" % (alternativeImportLine, moduleName),
                               globals(), locals(), ['__doc__']), "__doc__"))
            moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
                              (alternativeImportLine, moduleName, moduleName))

        # add the module to the step, and instance it
        stepDef.addModule(moduleDef)
        stepDef.createModuleInstance(module_type=moduleName, name=moduleName)

    # add parameters to the module definition
    for pName, pType, pValue, pDesc in parametersList:
        p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
        stepDef.addParameter(Parameter(parameter=p))

    return stepDef
示例#10
0
 def _getUserOutputDataModule(self):
   """ This is separated as not all applications require user specific output data (i.e. GetSRMFile and Overlay). Only used in UserJobs.
   
   The UserJobFinalization only runs last. It's called every step, but is running only if last.
   """
   moduledefinition = ModuleDefinition('UserJobFinalization')
   moduledefinition.setDescription('Uploads user output data files with specific policies.')
   body = 'from %s.%s import %s\n' % (self._importLocation, 'UserJobFinalization', 'UserJobFinalization')
   moduledefinition.setBody(body)
   return moduledefinition
示例#11
0
 def _getUserOutputDataModule(self):
     """ This is separated as not all applications require user specific output data (i.e. GetSRMFile and Overlay). 
     Only used in UserJobs.
     
     The UserJobFinalization only runs last. It's called every step, but is running only if last.
     """
     moduledefinition = ModuleDefinition('UserJobFinalization')
     moduledefinition.setDescription('Uploads user output data files with specific policies.')
     body = 'from %s.%s import %s\n' % (self._importLocation, 'UserJobFinalization', 'UserJobFinalization')
     moduledefinition.setBody(body)
     return moduledefinition
示例#12
0
    def test__getStepDefinition(self):
        importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
        # modules
        gaudiApp = ModuleDefinition("Script")
        body = importLine.replace("<MODULE>", "Script")
        gaudiApp.setDescription(
            getattr(
                __import__("%s.%s" % ("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), "__doc__"
            )
        )
        gaudiApp.setBody(body)

        genBKReport = ModuleDefinition("FailoverRequest")
        body = importLine.replace("<MODULE>", "FailoverRequest")
        genBKReport.setDescription(
            getattr(
                __import__("%s.%s" % ("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"]),
                "__doc__",
            )
        )
        genBKReport.setBody(body)

        # step
        appDefn = StepDefinition("App_Step")
        appDefn.addModule(gaudiApp)
        appDefn.createModuleInstance("Script", "Script")
        appDefn.addModule(genBKReport)
        appDefn.createModuleInstance("FailoverRequest", "FailoverRequest")

        appDefn.addParameterLinked(gaudiApp.parameters)

        stepDef = getStepDefinition("App_Step", ["Script", "FailoverRequest"])

        self.assertTrue(str(appDefn) == str(stepDef))

        self.job._addParameter(appDefn, "name", "type", "value", "desc")
        self.job._addParameter(appDefn, "name1", "type1", "value1", "desc1")

        stepDef = getStepDefinition(
            "App_Step",
            ["Script", "FailoverRequest"],
            parametersList=[["name", "type", "value", "desc"], ["name1", "type1", "value1", "desc1"]],
        )

        self.assertTrue(str(appDefn) == str(stepDef))
示例#13
0
    def _addRealFinalization(self):
        """ This is called at creation: now that the workflow is created at the last minute,
    we need to add this also at the last minute
    """
        importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'

        dataUpload = ModuleDefinition('UploadOutputData')
        dataUpload.setDescription('Uploads the output data')
        self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
        body = importLine.replace('<MODULE>', 'UploadOutputData')
        dataUpload.setBody(body)

        failoverRequest = ModuleDefinition('FailoverRequest')
        failoverRequest.setDescription('Sends any failover requests')
        self._addParameter(failoverRequest, 'enable', 'bool', False,
                           'EnableFlag')
        body = importLine.replace('<MODULE>', 'FailoverRequest')
        failoverRequest.setBody(body)

        registerdata = ModuleDefinition('RegisterOutputData')
        registerdata.setDescription(
            'Module to add in the metadata catalog the relevant info about the files'
        )
        self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
        body = importLine.replace('<MODULE>', 'RegisterOutputData')
        registerdata.setBody(body)

        logUpload = ModuleDefinition('UploadLogFile')
        logUpload.setDescription('Uploads the output log files')
        self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
        body = importLine.replace('<MODULE>', 'UploadLogFile')
        logUpload.setBody(body)

        finalization = StepDefinition('Job_Finalization')
        finalization.addModule(dataUpload)
        up = finalization.createModuleInstance('UploadOutputData',
                                               'dataUpload')
        up.setValue("enable", self.finalsdict['uploadData'])

        finalization.addModule(registerdata)
        ro = finalization.createModuleInstance('RegisterOutputData',
                                               'RegisterOutputData')
        ro.setValue("enable", self.finalsdict['registerData'])

        finalization.addModule(logUpload)
        ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
        ul.setValue("enable", self.finalsdict['uploadLog'])

        finalization.addModule(failoverRequest)
        fr = finalization.createModuleInstance('FailoverRequest',
                                               'failoverRequest')
        fr.setValue("enable", self.finalsdict['sendFailover'])

        self.workflow.addStep(finalization)
        self.workflow.createStepInstance('Job_Finalization', 'finalization')

        return S_OK()
    def _addRealFinalization(self):
        """ See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
        """
        importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
        dataUpload = ModuleDefinition('UploadOutputData')
        dataUpload.setDescription('Uploads the output data')
        self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
        body = string.replace(importLine, '<MODULE>', 'UploadOutputData')
        dataUpload.setBody(body)

        failoverRequest = ModuleDefinition('FailoverRequest')
        failoverRequest.setDescription('Sends any failover requests')
        self._addParameter(failoverRequest, 'enable', 'bool', False,
                           'EnableFlag')
        body = string.replace(importLine, '<MODULE>', 'FailoverRequest')
        failoverRequest.setBody(body)

        registerdata = ModuleDefinition('ILDRegisterOutputData')
        registerdata.setDescription(
            'Module to add in the metadata catalog the relevant info about the files'
        )
        self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
        body = string.replace(importLine, '<MODULE>', 'ILDRegisterOutputData')
        registerdata.setBody(body)

        logUpload = ModuleDefinition('UploadLogFile')
        logUpload.setDescription('Uploads the output log files')
        self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
        body = string.replace(importLine, '<MODULE>', 'UploadLogFile')
        logUpload.setBody(body)

        finalization = StepDefinition('Job_Finalization')
        finalization.addModule(dataUpload)
        up = finalization.createModuleInstance('UploadOutputData',
                                               'dataUpload')
        up.setValue("enable", self.finalsdict['uploadData'])

        finalization.addModule(registerdata)
        # TODO: create ILDRegisterOutputData
        ro = finalization.createModuleInstance('ILDRegisterOutputData',
                                               'ILDRegisterOutputData')
        ro.setValue("enable", self.finalsdict['registerData'])

        finalization.addModule(logUpload)
        ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
        ul.setValue("enable", self.finalsdict['uploadLog'])

        finalization.addModule(failoverRequest)
        fr = finalization.createModuleInstance('FailoverRequest',
                                               'failoverRequest')
        fr.setValue("enable", self.finalsdict['sendFailover'])

        self.workflow.addStep(finalization)
        self.workflow.createStepInstance('Job_Finalization', 'finalization')

        return S_OK()
示例#15
0
  def addFinalization(self, uploadData=False, registerData=False, uploadLog = False, sendFailover=False):
    """ Add finalization step

    @param uploadData: Upload or not the data to the storage
    @param uploadLog: Upload log file to storage (currently only available for admins, thus add them to OutputSandbox)
    @param sendFailover: Send Failover requests, and declare files as processed or unused in transfDB
    @param registerData: Register data in the file catalog
    @todo: Do the registration only once, instead of once for each job

    """
    self.importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
    dataUpload = ModuleDefinition('UploadOutputData')
    dataUpload.setDescription('Uploads the output data')
    self._addParameter(dataUpload,'enable','bool',False,'EnableFlag')
    body = string.replace(self.importLine,'<MODULE>','UploadOutputData')
    dataUpload.setBody(body)

    failoverRequest = ModuleDefinition('FailoverRequest')
    failoverRequest.setDescription('Sends any failover requests')
    self._addParameter(failoverRequest,'enable','bool',False,'EnableFlag')
    body = string.replace(self.importLine,'<MODULE>','FailoverRequest')
    failoverRequest.setBody(body)

    registerdata = ModuleDefinition('DBDGenRegisterOutputData')
    registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files')
    self._addParameter(registerdata,'enable','bool',False,'EnableFlag')
    body = string.replace(self.importLine,'<MODULE>','DBDGenRegisterOutputData')
    registerdata.setBody(body)

    logUpload = ModuleDefinition('UploadLogFile')
    logUpload.setDescription('Uploads the output log files')
    self._addParameter(logUpload,'enable','bool',False,'EnableFlag')
    body = string.replace(self.importLine,'<MODULE>','UploadLogFile')
    logUpload.setBody(body)

    finalization = StepDefinition('Job_Finalization')
    finalization.addModule(dataUpload)
    up = finalization.createModuleInstance('UploadOutputData','dataUpload')
    up.setValue("enable",uploadData)

    finalization.addModule(registerdata)
    ro = finalization.createModuleInstance('DBDGenRegisterOutputData','DBDGenRegisterOutputData')
    ro.setValue("enable",registerData)

    finalization.addModule(logUpload)
    ul  = finalization.createModuleInstance('UploadLogFile','logUpload')
    ul.setValue("enable",uploadLog)

    finalization.addModule(failoverRequest)
    fr = finalization.createModuleInstance('FailoverRequest','failoverRequest')
    fr.setValue("enable",sendFailover)
    
    self.workflow.addStep(finalization)
    self.workflow.createStepInstance('Job_Finalization', 'finalization')

    return S_OK()
示例#16
0
    def test__getStepDefinition(self):
        importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
        # modules
        gaudiApp = ModuleDefinition('Script')
        body = importLine.replace('<MODULE>', 'Script')
        gaudiApp.setDescription(
            getattr(
                __import__("%s.%s" % ('DIRAC.Workflow.Modules', 'Script'),
                           globals(), locals(), ['__doc__']), "__doc__"))
        gaudiApp.setBody(body)

        genBKReport = ModuleDefinition('FailoverRequest')
        body = importLine.replace('<MODULE>', 'FailoverRequest')
        genBKReport.setDescription(
            getattr(
                __import__(
                    "%s.%s" % ('DIRAC.Workflow.Modules', 'FailoverRequest'),
                    globals(), locals(), ['__doc__']), "__doc__"))
        genBKReport.setBody(body)

        # step
        appDefn = StepDefinition('App_Step')
        appDefn.addModule(gaudiApp)
        appDefn.createModuleInstance('Script', 'Script')
        appDefn.addModule(genBKReport)
        appDefn.createModuleInstance('FailoverRequest', 'FailoverRequest')

        appDefn.addParameterLinked(gaudiApp.parameters)

        stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'])

        self.assert_(str(appDefn) == str(stepDef))

        self.job._addParameter(appDefn, 'name', 'type', 'value', 'desc')
        self.job._addParameter(appDefn, 'name1', 'type1', 'value1', 'desc1')

        stepDef = getStepDefinition(
            'App_Step', ['Script', 'FailoverRequest'],
            parametersList=[['name', 'type', 'value', 'desc'],
                            ['name1', 'type1', 'value1', 'desc1']])

        self.assert_(str(appDefn) == str(stepDef))
示例#17
0
文件: Job.py 项目: mesillo/DIRAC
 def __getScriptStep( self, name = 'Script' ):
   """Internal function. This method controls the definition for a script module.
   """
   # Create the script module first
   moduleName = 'Script'
   module = ModuleDefinition( moduleName )
   module.setDescription( 'A  script module that can execute any provided script.' )
   body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n'
   module.setBody( body )
   # Create Step definition
   step = StepDefinition( name )
   step.addModule( module )
   moduleInstance = step.createModuleInstance( 'Script', name )
   # Define step parameters
   step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) )
   step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) )
   step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) )
   step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
   return step
示例#18
0
  def test__getStepDefinition( self ):
    importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
    # modules
    gaudiApp = ModuleDefinition( 'Script' )
    body = importLine.replace( '<MODULE>', 'Script' )
    gaudiApp.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'Script' ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
    gaudiApp.setBody( body )

    genBKReport = ModuleDefinition( 'FailoverRequest' )
    body = importLine.replace( '<MODULE>', 'FailoverRequest' )
    genBKReport.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'FailoverRequest' ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
    genBKReport.setBody( body )

    # step
    appDefn = StepDefinition( 'App_Step' )
    appDefn.addModule( gaudiApp )
    appDefn.createModuleInstance( 'Script', 'Script' )
    appDefn.addModule( genBKReport )
    appDefn.createModuleInstance( 'FailoverRequest', 'FailoverRequest' )

    appDefn.addParameterLinked( gaudiApp.parameters )

    stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'] )

    self.assertTrue( str( appDefn ) == str( stepDef ) )



    self.job._addParameter( appDefn, 'name', 'type', 'value', 'desc' )
    self.job._addParameter( appDefn, 'name1', 'type1', 'value1', 'desc1' )


    stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'],
                                 parametersList = [[ 'name', 'type', 'value', 'desc' ],
                                                   [ 'name1', 'type1', 'value1', 'desc1' ]] )

    self.assertTrue( str( appDefn ) == str( stepDef ) )
示例#19
0
def getStepDefinition( stepName, modulesNameList = [], importLine = """""", parametersList = [] ):
  """ Given a name, a list of modules name, and a list of parameters, returns a step definition.
      Remember that Step definition = Parameters + Module Instances
  """

  # In case the importLine is not set, this is looking for a DIRAC extension, if any.
  # The extension is supposed to be called ExtDIRAC.
  if not importLine:
    importLine = "DIRAC.Workflow.Modules"
    for ext in getCSExtensions():
      if ext.lower() == getVO():
        importLine = ext + "DIRAC.Workflow.Modules"
        break

  stepDef = StepDefinition( stepName )

  for moduleName in modulesNameList:

    # create the module definition
    moduleDef = ModuleDefinition( moduleName )
    try:
      # Look in the importLine given, or the DIRAC if the given location can't be imported
      moduleDef.setDescription( getattr( __import__( "%s.%s" % ( importLine, moduleName ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
      moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( importLine, moduleName, moduleName ) )
    except ImportError:
      alternativeImportLine = "DIRAC.Workflow.Modules"
      moduleDef.setDescription( getattr( __import__( "%s.%s" % ( alternativeImportLine, moduleName ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
      moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( alternativeImportLine, moduleName, moduleName ) )

    # add the module to the step, and instance it
    stepDef.addModule( moduleDef )
    stepDef.createModuleInstance( module_type = moduleName, name = moduleName )

  # add parameters to the module definition
  for pName, pType, pValue, pDesc in parametersList:
    p = Parameter( pName, pValue, pType, "", "", True, False, pDesc )
    stepDef.addParameter( Parameter( parameter = p ) )

  return stepDef
示例#20
0
文件: Job.py 项目: sposs/DIRAC
 def __getScriptStep( self, name = 'Script' ):
   """Internal function. This method controls the definition for a script module.
   """
   # Create the script module first
   moduleName = 'Script'
   module = ModuleDefinition( moduleName )
   module.setDescription( 'A  script module that can execute any provided script.' )
   body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n'
   module.setBody( body )
   # Create Step definition
   step = StepDefinition( name )
   step.addModule( module )
   moduleInstance = step.createModuleInstance( 'Script', name )
   # Define step parameters
   step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) )
   step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) )
   step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) )
   step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
   return step
示例#21
0
def getStepDefinition(stepName,
                      modulesNameList=None,
                      importLine="",
                      parametersList=None):
    """Given a name, a list of modules name, and a list of parameters, returns a step definition.
    Remember that Step definition = Parameters + Module Instances
    """
    if modulesNameList is None:
        modulesNameList = []

    if parametersList is None:
        parametersList = []

    stepDef = StepDefinition(stepName)

    for moduleName in modulesNameList:
        module = None
        if importLine:
            try:
                module = importlib.import_module(importLine + "." + moduleName)
            except ImportError:
                pass
        # In case the importLine is not set, this is looking for a DIRAC extension, if any
        if module is None:
            module = ObjectLoader().loadModule("Workflow.Modules." +
                                               moduleName)["Value"]

        # create the module definition
        moduleDef = ModuleDefinition(moduleName)
        moduleDef.setDescription(module.__doc__)
        moduleDef.setBody("\nfrom %s import %s\n" %
                          (module.__name__, moduleName))

        # add the module to the step, and instance it
        stepDef.addModule(moduleDef)
        stepDef.createModuleInstance(module_type=moduleName, name=moduleName)

    # add parameters to the module definition
    for pName, pType, pValue, pDesc in parametersList:
        p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
        stepDef.addParameter(Parameter(parameter=p))

    return stepDef