Esempio n. 1
0
    def addWrapper(self, logFile = ''):
        """ Overload the DIRAC.Job.setExecutable
        """
        logFile = str(logFile)
        stepDefn = 'WrapperStep'
        stepName = 'RunWrapperStep'

        moduleName = 'GlastWrapperCall'
        module = ModuleDefinition( moduleName )
        module.setDescription( 'The utility that calls the pipeline_wrapper.' )
        body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
        module.setBody( body )
        # Create Step definition
        step = StepDefinition( stepDefn )
        step.addModule( module )
        #moduleInstance = step.createModuleInstance( 'GlastWrapperCall', stepDefn )
        # Define step parameters
        step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
        self.addToOutputSandbox.append( logFile )
        self.workflow.addStep( step )

        # Define Step and its variables  
        stepInstance = self.workflow.createStepInstance( stepDefn, stepName )
        stepInstance.setValue( "logFile", logFile )

        return S_OK()
Esempio n. 2
0
  def _addToWorkflow(self):
    """ This is called just before submission. It creates the actual workflow. 
    The linking of parameters can only be done here
    """
    for application in self.applicationlist:
      #Start by defining step number 
      self.stepCount += 1
      
      res = application._analyseJob(self)
      if not res['OK']:
        return res
    
      res = application._checkWorkflowConsistency()
      if not res['OK']:
        self.log.error("%s failed to check its consistency:" % application, "%s" % res['Message'])
        return S_ERROR("%s failed to check its consistency: %s" % (application, res['Message']))
      
      ##Now we can create the step and add it to the workflow
      #First we need a unique name, let's use the application name and step number
      stepname = "%s_step_%s" % (application.appname, self.stepCount)
      stepdefinition = StepDefinition(stepname)
      self.steps.append(stepdefinition)
      
      ##Set the modules needed by the application
      res = self._jobSpecificModules(application, stepdefinition)
      if not res['OK']:
        self.log.error("Failed to add modules:", "%s" % res['Message'])
        return S_ERROR("Failed to add modules: %s" % res['Message'])
  
      ### add the parameters to  the step
      res = application._addParametersToStep(stepdefinition)
      if not res['OK']:
        self.log.error("Failed to add parameters:", "%s" % res['Message'])   
        return S_ERROR("Failed to add parameters: %s" % res['Message'])   
      
      ##Now the step is defined, let's add it to the workflow
      self.workflow.addStep(stepdefinition)
    
      ###Now we need to get a step instance object to set the parameters' values
      stepInstance = self.workflow.createStepInstance(stepdefinition.getType(), stepname)

      ##Set the parameters values to the step instance
      res = application._setStepParametersValues(stepInstance)
      if not res['OK']:
        self.log.error("Failed to resolve parameters values:", "%s" % res['Message']) 
        return S_ERROR("Failed to resolve parameters values: %s" % res['Message'])   
    
      res = application._resolveLinkedStepParameters(stepInstance)
      if not res['OK']:
        self.log.error("Failed to resolve linked parameters:", "%s" % res['Message'])
        return S_ERROR("Failed to resolve linked parameters: %s" % res['Message'])
      #Now prevent overwriting of parameter values.
      application._addedtojob()
  
      self._addParameter(self.workflow, 'TotalSteps', 'String', self.stepCount, 'Total number of steps')
      
    return S_OK()
Esempio n. 3
0
def getStepDefinition(stepName,
                      modulesNameList=None,
                      importLine="""""",
                      parametersList=None):
    """ Given a name, a list of modules name, and a list of parameters, returns a step definition.
      Remember that Step definition = Parameters + Module Instances
  """

    if modulesNameList is None:
        modulesNameList = []

    if parametersList is None:
        parametersList = []

    # In case the importLine is not set, this is looking for a DIRAC extension, if any.
    # The extension is supposed to be called ExtDIRAC.
    if not importLine:
        importLine = "DIRAC.Workflow.Modules"
        for ext in getCSExtensions():
            if ext.lower() == getVO():
                importLine = ext + "DIRAC.Workflow.Modules"
                break

    stepDef = StepDefinition(stepName)

    for moduleName in modulesNameList:

        # create the module definition
        moduleDef = ModuleDefinition(moduleName)
        try:
            # Look in the importLine given, or the DIRAC if the given location can't be imported
            moduleDef.setDescription(
                getattr(
                    __import__("%s.%s" % (importLine, moduleName), globals(),
                               locals(), ['__doc__']), "__doc__"))
            moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
                              (importLine, moduleName, moduleName))
        except ImportError:
            alternativeImportLine = "DIRAC.Workflow.Modules"
            moduleDef.setDescription(
                getattr(
                    __import__("%s.%s" % (alternativeImportLine, moduleName),
                               globals(), locals(), ['__doc__']), "__doc__"))
            moduleDef.setBody("""\nfrom %s.%s import %s\n""" %
                              (alternativeImportLine, moduleName, moduleName))

        # add the module to the step, and instance it
        stepDef.addModule(moduleDef)
        stepDef.createModuleInstance(module_type=moduleName, name=moduleName)

    # add parameters to the module definition
    for pName, pType, pValue, pDesc in parametersList:
        p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
        stepDef.addParameter(Parameter(parameter=p))

    return stepDef
Esempio n. 4
0
    def addWrapper(self, logFile=''):
        """ Overload the DIRAC.Job.setExecutable
        """
        logFile = str(logFile)
        stepDefn = 'WrapperStep'
        stepName = 'RunWrapperStep'

        moduleName = 'GlastWrapperCall'
        module = ModuleDefinition(moduleName)
        module.setDescription('The utility that calls the pipeline_wrapper.')
        body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n'
        module.setBody(body)
        # Create Step definition
        step = StepDefinition(stepDefn)
        step.addModule(module)
        moduleInstance = step.createModuleInstance('GlastWrapperCall',
                                                   stepDefn)
        # Define step parameters
        step.addParameter(
            Parameter("logFile", "", "string", "", "", False, False,
                      'Log file name'))
        self.addToOutputSandbox.append(logFile)
        self.workflow.addStep(step)

        # Define Step and its variables
        stepInstance = self.workflow.createStepInstance(stepDefn, stepName)
        stepInstance.setValue("logFile", logFile)

        return S_OK()
Esempio n. 5
0
def createSingleModuleWorkflow(module, name):
    ''' Creates a workflow based on a single module definition
  '''

    moduleType = module.getType()
    moduleName = name

    workflow = Workflow()
    step = StepDefinition(moduleType + '_step')
    step.addModule(module)
    moduleInstance = step.createModuleInstance(moduleType, moduleName)

    step.addParameter(moduleInstance.parameters.getInput())
    workflow.addParameter(moduleInstance.parameters.getInput())

    workflow.addStep(step)
    stepInstance = workflow.createStepInstance(moduleType + '_step',
                                               moduleName + '_step')

    # Propagate the module input parameters to the workflow level
    moduleInstance.linkParameterUp(moduleInstance.parameters.getInput())
    stepInstance.linkParameterUp(moduleInstance.parameters.getInput())

    workflow.setName(name)
    workflow.setDescription('Single module workflow from ' + moduleType +
                            ' type module')
    workflow.setDescrShort(moduleType + ' workflow')
    return workflow
Esempio n. 6
0
def createSingleModuleWorkflow(module,name):
  ''' Creates a workflow based on a single module definition
  '''

  moduleType = module.getType()
  moduleName = name

  workflow = Workflow()
  step = StepDefinition(moduleType+'_step')
  step.addModule(module)
  moduleInstance = step.createModuleInstance(moduleType,moduleName)

  step.addParameter(moduleInstance.parameters.getInput())
  workflow.addParameter(moduleInstance.parameters.getInput())

  workflow.addStep(step)
  stepInstance = workflow.createStepInstance(moduleType+'_step',moduleName+'_step')

  # Propagate the module input parameters to the workflow level
  moduleInstance.linkParameterUp(moduleInstance.parameters.getInput())
  stepInstance.linkParameterUp(moduleInstance.parameters.getInput())

  workflow.setName(name)
  workflow.setDescription('Single module workflow from '+moduleType+' type module')
  workflow.setDescrShort(moduleType+' workflow')
  return workflow
Esempio n. 7
0
def getStepDefinition(stepName,
                      modulesNameList=None,
                      importLine="",
                      parametersList=None):
    """Given a name, a list of modules name, and a list of parameters, returns a step definition.
    Remember that Step definition = Parameters + Module Instances
    """
    if modulesNameList is None:
        modulesNameList = []

    if parametersList is None:
        parametersList = []

    stepDef = StepDefinition(stepName)

    for moduleName in modulesNameList:
        module = None
        if importLine:
            try:
                module = importlib.import_module(importLine + "." + moduleName)
            except ImportError:
                pass
        # In case the importLine is not set, this is looking for a DIRAC extension, if any
        if module is None:
            module = ObjectLoader().loadModule("Workflow.Modules." +
                                               moduleName)["Value"]

        # create the module definition
        moduleDef = ModuleDefinition(moduleName)
        moduleDef.setDescription(module.__doc__)
        moduleDef.setBody("\nfrom %s import %s\n" %
                          (module.__name__, moduleName))

        # add the module to the step, and instance it
        stepDef.addModule(moduleDef)
        stepDef.createModuleInstance(module_type=moduleName, name=moduleName)

    # add parameters to the module definition
    for pName, pType, pValue, pDesc in parametersList:
        p = Parameter(pName, pValue, pType, "", "", True, False, pDesc)
        stepDef.addParameter(Parameter(parameter=p))

    return stepDef
Esempio n. 8
0
    def startElement(self, name, attrs):
        # print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames()
        self.clearCharacters(
        )  # clear to remove empty or nonprintable characters

        if name == "Workflow":
            if self.root == None:  # if root not defined by constractor
                self.root = Workflow()
            self.stack.append(self.root)

        elif name == "StepDefinition":
            obj = StepDefinition("TemporaryXMLObject_StepDefinition")
            if self.root == None:  # in case we are saving Step only
                self.root = obj
            self.stack.append(obj)

        elif name == "StepInstance":
            obj = StepInstance("TemporaryXMLObject_StepInstance")
            self.stack.append(obj)

        elif name == "ModuleDefinition":
            obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition")
            if self.root == None:  # in case we are saving Module only
                self.root = obj
            self.stack.append(obj)

        elif name == "ModuleInstance":
            obj = ModuleInstance("TemporaryXMLObject_ModuleInstance")
            self.stack.append(obj)

        elif name == "Parameter":
            obj = Parameter(str(attrs['name']), None, str(attrs['type']),
                            str(attrs['linked_module']),
                            str(attrs['linked_parameter']), str(attrs['in']),
                            str(attrs['out']), str(attrs['description']))
            self.stack.append(obj)

        # TEMPORARY CODE
        elif name == "origin" or name == "version" or name == "name" or name == "type" or name == "value" or\
        name == "required" or name == "descr_short" or name == "name" or name == "type"  or name == "description"  or name == "body":
            pass
        else:
            print "UNTREATED! startElement name=", name, "attr=", attrs.getLength(
            ), attrs.getNames()
            pass
Esempio n. 9
0
def getStepDefinition( stepName, modulesNameList = [], importLine = """""", parametersList = [] ):
  """ Given a name, a list of modules name, and a list of parameters, returns a step definition.
      Remember that Step definition = Parameters + Module Instances
  """

  # In case the importLine is not set, this is looking for a DIRAC extension, if any.
  # The extension is supposed to be called ExtDIRAC.
  if not importLine:
    importLine = "DIRAC.Workflow.Modules"
    for ext in getCSExtensions():
      if ext.lower() == getVO():
        importLine = ext + "DIRAC.Workflow.Modules"
        break

  stepDef = StepDefinition( stepName )

  for moduleName in modulesNameList:

    # create the module definition
    moduleDef = ModuleDefinition( moduleName )
    try:
      # Look in the importLine given, or the DIRAC if the given location can't be imported
      moduleDef.setDescription( getattr( __import__( "%s.%s" % ( importLine, moduleName ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
      moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( importLine, moduleName, moduleName ) )
    except ImportError:
      alternativeImportLine = "DIRAC.Workflow.Modules"
      moduleDef.setDescription( getattr( __import__( "%s.%s" % ( alternativeImportLine, moduleName ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
      moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( alternativeImportLine, moduleName, moduleName ) )

    # add the module to the step, and instance it
    stepDef.addModule( moduleDef )
    stepDef.createModuleInstance( module_type = moduleName, name = moduleName )

  # add parameters to the module definition
  for pName, pType, pValue, pDesc in parametersList:
    p = Parameter( pName, pValue, pType, "", "", True, False, pDesc )
    stepDef.addParameter( Parameter( parameter = p ) )

  return stepDef
    def _addRealFinalization(self):
        """ See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
        """
        importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
        dataUpload = ModuleDefinition('UploadOutputData')
        dataUpload.setDescription('Uploads the output data')
        self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
        body = string.replace(importLine, '<MODULE>', 'UploadOutputData')
        dataUpload.setBody(body)

        failoverRequest = ModuleDefinition('FailoverRequest')
        failoverRequest.setDescription('Sends any failover requests')
        self._addParameter(failoverRequest, 'enable', 'bool', False,
                           'EnableFlag')
        body = string.replace(importLine, '<MODULE>', 'FailoverRequest')
        failoverRequest.setBody(body)

        registerdata = ModuleDefinition('ILDRegisterOutputData')
        registerdata.setDescription(
            'Module to add in the metadata catalog the relevant info about the files'
        )
        self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
        body = string.replace(importLine, '<MODULE>', 'ILDRegisterOutputData')
        registerdata.setBody(body)

        logUpload = ModuleDefinition('UploadLogFile')
        logUpload.setDescription('Uploads the output log files')
        self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
        body = string.replace(importLine, '<MODULE>', 'UploadLogFile')
        logUpload.setBody(body)

        finalization = StepDefinition('Job_Finalization')
        finalization.addModule(dataUpload)
        up = finalization.createModuleInstance('UploadOutputData',
                                               'dataUpload')
        up.setValue("enable", self.finalsdict['uploadData'])

        finalization.addModule(registerdata)
        # TODO: create ILDRegisterOutputData
        ro = finalization.createModuleInstance('ILDRegisterOutputData',
                                               'ILDRegisterOutputData')
        ro.setValue("enable", self.finalsdict['registerData'])

        finalization.addModule(logUpload)
        ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
        ul.setValue("enable", self.finalsdict['uploadLog'])

        finalization.addModule(failoverRequest)
        fr = finalization.createModuleInstance('FailoverRequest',
                                               'failoverRequest')
        fr.setValue("enable", self.finalsdict['sendFailover'])

        self.workflow.addStep(finalization)
        self.workflow.createStepInstance('Job_Finalization', 'finalization')

        return S_OK()
Esempio n. 11
0
  def test__getStepDefinition( self ):
    importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
    # modules
    gaudiApp = ModuleDefinition( 'Script' )
    body = importLine.replace( '<MODULE>', 'Script' )
    gaudiApp.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'Script' ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
    gaudiApp.setBody( body )

    genBKReport = ModuleDefinition( 'FailoverRequest' )
    body = importLine.replace( '<MODULE>', 'FailoverRequest' )
    genBKReport.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'FailoverRequest' ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
    genBKReport.setBody( body )

    # step
    appDefn = StepDefinition( 'App_Step' )
    appDefn.addModule( gaudiApp )
    appDefn.createModuleInstance( 'Script', 'Script' )
    appDefn.addModule( genBKReport )
    appDefn.createModuleInstance( 'FailoverRequest', 'FailoverRequest' )

    appDefn.addParameterLinked( gaudiApp.parameters )

    stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'] )

    self.assertTrue( str( appDefn ) == str( stepDef ) )



    self.job._addParameter( appDefn, 'name', 'type', 'value', 'desc' )
    self.job._addParameter( appDefn, 'name1', 'type1', 'value1', 'desc1' )


    stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'],
                                 parametersList = [[ 'name', 'type', 'value', 'desc' ],
                                                   [ 'name1', 'type1', 'value1', 'desc1' ]] )

    self.assertTrue( str( appDefn ) == str( stepDef ) )
Esempio n. 12
0
    def _addRealFinalization(self):
        """ This is called at creation: now that the workflow is created at the last minute,
    we need to add this also at the last minute
    """
        importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'

        dataUpload = ModuleDefinition('UploadOutputData')
        dataUpload.setDescription('Uploads the output data')
        self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
        body = importLine.replace('<MODULE>', 'UploadOutputData')
        dataUpload.setBody(body)

        failoverRequest = ModuleDefinition('FailoverRequest')
        failoverRequest.setDescription('Sends any failover requests')
        self._addParameter(failoverRequest, 'enable', 'bool', False,
                           'EnableFlag')
        body = importLine.replace('<MODULE>', 'FailoverRequest')
        failoverRequest.setBody(body)

        registerdata = ModuleDefinition('RegisterOutputData')
        registerdata.setDescription(
            'Module to add in the metadata catalog the relevant info about the files'
        )
        self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
        body = importLine.replace('<MODULE>', 'RegisterOutputData')
        registerdata.setBody(body)

        logUpload = ModuleDefinition('UploadLogFile')
        logUpload.setDescription('Uploads the output log files')
        self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
        body = importLine.replace('<MODULE>', 'UploadLogFile')
        logUpload.setBody(body)

        finalization = StepDefinition('Job_Finalization')
        finalization.addModule(dataUpload)
        up = finalization.createModuleInstance('UploadOutputData',
                                               'dataUpload')
        up.setValue("enable", self.finalsdict['uploadData'])

        finalization.addModule(registerdata)
        ro = finalization.createModuleInstance('RegisterOutputData',
                                               'RegisterOutputData')
        ro.setValue("enable", self.finalsdict['registerData'])

        finalization.addModule(logUpload)
        ul = finalization.createModuleInstance('UploadLogFile', 'logUpload')
        ul.setValue("enable", self.finalsdict['uploadLog'])

        finalization.addModule(failoverRequest)
        fr = finalization.createModuleInstance('FailoverRequest',
                                               'failoverRequest')
        fr.setValue("enable", self.finalsdict['sendFailover'])

        self.workflow.addStep(finalization)
        self.workflow.createStepInstance('Job_Finalization', 'finalization')

        return S_OK()
Esempio n. 13
0
File: Job.py Progetto: sposs/DIRAC
 def __getScriptStep( self, name = 'Script' ):
   """Internal function. This method controls the definition for a script module.
   """
   # Create the script module first
   moduleName = 'Script'
   module = ModuleDefinition( moduleName )
   module.setDescription( 'A  script module that can execute any provided script.' )
   body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n'
   module.setBody( body )
   # Create Step definition
   step = StepDefinition( name )
   step.addModule( module )
   moduleInstance = step.createModuleInstance( 'Script', name )
   # Define step parameters
   step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) )
   step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) )
   step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) )
   step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
   return step
Esempio n. 14
0
    def test__getStepDefinition(self):
        importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
        # modules
        gaudiApp = ModuleDefinition("Script")
        body = importLine.replace("<MODULE>", "Script")
        gaudiApp.setDescription(
            getattr(
                __import__("%s.%s" % ("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), "__doc__"
            )
        )
        gaudiApp.setBody(body)

        genBKReport = ModuleDefinition("FailoverRequest")
        body = importLine.replace("<MODULE>", "FailoverRequest")
        genBKReport.setDescription(
            getattr(
                __import__("%s.%s" % ("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"]),
                "__doc__",
            )
        )
        genBKReport.setBody(body)

        # step
        appDefn = StepDefinition("App_Step")
        appDefn.addModule(gaudiApp)
        appDefn.createModuleInstance("Script", "Script")
        appDefn.addModule(genBKReport)
        appDefn.createModuleInstance("FailoverRequest", "FailoverRequest")

        appDefn.addParameterLinked(gaudiApp.parameters)

        stepDef = getStepDefinition("App_Step", ["Script", "FailoverRequest"])

        self.assertTrue(str(appDefn) == str(stepDef))

        self.job._addParameter(appDefn, "name", "type", "value", "desc")
        self.job._addParameter(appDefn, "name1", "type1", "value1", "desc1")

        stepDef = getStepDefinition(
            "App_Step",
            ["Script", "FailoverRequest"],
            parametersList=[["name", "type", "value", "desc"], ["name1", "type1", "value1", "desc1"]],
        )

        self.assertTrue(str(appDefn) == str(stepDef))
Esempio n. 15
0
  def append(self,application):
    """ Helper function
    
    This is the main part: call for every application
    @param application: Application instance
    
    """
    #Start by defining step number
    self.stepnumber = len(self.steps) + 1
    

    res = application._analyseJob(self)
    if not res['OK']:
      return res
    
    res = application._checkConsistency()
    if not res['OK']:
      self.log.error("%s failed to check its consistency: %s"%(application,res['Message']))
      return S_ERROR("%s failed to check its consistency: %s"%(application,res['Message']))
    
    res = self._jobSpecificParams(application)
    if not res['OK']:
      self.log.error("%s failed job specific checks: %s"%(application,res['Message']))
      return S_ERROR("%s failed job specific checks: %s"%(application,res['Message']))

    res = application._checkFinalConsistency()
    if not res['OK']:
      self.log.error("%s failed to check its consistency: %s"%(application,res['Message']))
      return S_ERROR("%s failed to check its consistency: %s"%(application,res['Message']))
    
    ### Once the consistency has been checked, we can add the application to the list of apps.
    self.applicationlist.append(application)
    ##Get the application's sandbox and add it to the job's
    for isb in application.inputSB:
      if not isb in self.inputsandbox:
        self.inputsandbox.append(isb)
    #self.inputsandbox.extend(application.inputSB)

    ##Now we can create the step and add it to the workflow
    #First we need a unique name, let's use the application name and step number
    stepname = "%s_step_%s"%(application.appname,self.stepnumber)
    stepdefinition = StepDefinition(stepname)
    self.steps.append(stepdefinition)

    ##Set the modules needed by the application
    res = self._jobSpecificModules(application,stepdefinition)
    if not res['OK']:
      self.log.error("Failed to add modules: %s"%res['Message'])
      return S_ERROR("Failed to add modules: %s"%res['Message'])
  
    ### add the parameters to  the step
    res = application._addParametersToStep(stepdefinition)
    if not res['OK']:
      self.log.error("Failed to add parameters: %s"%res['Message'])   
      return S_ERROR("Failed to add parameters: %s"%res['Message'])   
      
    ##Now the step is defined, let's add it to the workflow
    self.workflow.addStep(stepdefinition)
    
    ###Now we need to get a step instance object to set the parameters' values
    stepInstance = self.workflow.createStepInstance(stepdefinition.getType(),stepname)

    ##Set the parameters values to the step instance
    res = application._setStepParametersValues(stepInstance)
    if not res['OK']:
      self.log.error("Failed to resolve parameters values: %s"%res['Message']) 
      return S_ERROR("Failed to resolve parameters values: %s"%res['Message'])   
    
    ##stepInstance.setLink("InputFile",here lies the step name of the linked step, maybe get it from the application,"OutputFile")
    res = application._resolveLinkedStepParameters(stepInstance)
    if not res['OK']:
      self.log.error("Failed to resolve linked parameters: %s"%res['Message'])
      return S_ERROR("Failed to resolve linked parameters: %s"%res['Message'])
    #Now prevent overwriting of parameter values.
    application._addedtojob()
  
    self._addParameter(self.workflow, 'TotalSteps', 'String', self.stepnumber, 'Total number of steps')
    if application.nbevts:
      self._addParameter(self.workflow, 'NbOfEvts', 'int', application.nbevts, "Number of events to process")
  
    ##Finally, add the software packages if needed
    if application.appname and application.version:
      self._addSoftware(application.appname, application.version)
      
    return S_OK()
Esempio n. 16
0
  def test__getStepDefinition(self):
    importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
    # modules
    gaudiApp = ModuleDefinition('Script')
    body = importLine.replace('<MODULE>', 'Script')
    gaudiApp.setDescription(getattr(__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'Script'),
                                               globals(), locals(), ['__doc__']),
                                    "__doc__"))
    gaudiApp.setBody(body)

    genBKReport = ModuleDefinition('FailoverRequest')
    body = importLine.replace('<MODULE>', 'FailoverRequest')
    genBKReport.setDescription(getattr(__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'FailoverRequest'),
                                                  globals(), locals(), ['__doc__']),
                                       "__doc__"))
    genBKReport.setBody(body)

    # step
    appDefn = StepDefinition('App_Step')
    appDefn.addModule(gaudiApp)
    appDefn.createModuleInstance('Script', 'Script')
    appDefn.addModule(genBKReport)
    appDefn.createModuleInstance('FailoverRequest', 'FailoverRequest')

    appDefn.addParameterLinked(gaudiApp.parameters)

    stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'])

    self.assertTrue(str(appDefn) == str(stepDef))

    self.job._addParameter(appDefn, 'name', 'type', 'value', 'desc')
    self.job._addParameter(appDefn, 'name1', 'type1', 'value1', 'desc1')

    stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'],
                                parametersList=[['name', 'type', 'value', 'desc'],
                                                ['name1', 'type1', 'value1', 'desc1']])

    self.assertTrue(str(appDefn) == str(stepDef))
Esempio n. 17
0
    def _addToWorkflow(self):
        """ This is called just before submission. It creates the actual workflow. 
    The linking of parameters can only be done here
    """
        for application in self.applicationlist:
            #Start by defining step number
            self.stepCount += 1

            res = application._analyseJob(self)
            if not res['OK']:
                return res

            res = application._checkWorkflowConsistency()
            if not res['OK']:
                self.log.error(
                    "%s failed to check its consistency:" % application,
                    "%s" % res['Message'])
                return S_ERROR("%s failed to check its consistency: %s" %
                               (application, res['Message']))

            ##Now we can create the step and add it to the workflow
            #First we need a unique name, let's use the application name and step number
            stepname = "%s_step_%s" % (application.appname, self.stepCount)
            stepdefinition = StepDefinition(stepname)
            self.steps.append(stepdefinition)

            ##Set the modules needed by the application
            res = self._jobSpecificModules(application, stepdefinition)
            if not res['OK']:
                self.log.error("Failed to add modules:", "%s" % res['Message'])
                return S_ERROR("Failed to add modules: %s" % res['Message'])

            ### add the parameters to  the step
            res = application._addParametersToStep(stepdefinition)
            if not res['OK']:
                self.log.error("Failed to add parameters:",
                               "%s" % res['Message'])
                return S_ERROR("Failed to add parameters: %s" % res['Message'])

            ##Now the step is defined, let's add it to the workflow
            self.workflow.addStep(stepdefinition)

            ###Now we need to get a step instance object to set the parameters' values
            stepInstance = self.workflow.createStepInstance(
                stepdefinition.getType(), stepname)

            ##Set the parameters values to the step instance
            res = application._setStepParametersValues(stepInstance)
            if not res['OK']:
                self.log.error("Failed to resolve parameters values:",
                               "%s" % res['Message'])
                return S_ERROR("Failed to resolve parameters values: %s" %
                               res['Message'])

            res = application._resolveLinkedStepParameters(stepInstance)
            if not res['OK']:
                self.log.error("Failed to resolve linked parameters:",
                               "%s" % res['Message'])
                return S_ERROR("Failed to resolve linked parameters: %s" %
                               res['Message'])
            #Now prevent overwriting of parameter values.
            application._addedtojob()

            self._addParameter(self.workflow, 'TotalSteps', 'String',
                               self.stepCount, 'Total number of steps')

        return S_OK()
Esempio n. 18
0
    def _addRealFinalization( self ):
        """ See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition
        """
        importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
        dataUpload = ModuleDefinition( 'UploadOutputData' )
        dataUpload.setDescription( 'Uploads the output data' )
        self._addParameter( dataUpload, 'enable', 'bool', False, 'EnableFlag' )
        body = string.replace( importLine, '<MODULE>', 'UploadOutputData' )
        dataUpload.setBody( body )

        failoverRequest = ModuleDefinition( 'FailoverRequest' )
        failoverRequest.setDescription( 'Sends any failover requests' )
        self._addParameter( failoverRequest, 'enable', 'bool', False, 'EnableFlag' )
        body = string.replace( importLine, '<MODULE>', 'FailoverRequest' )
        failoverRequest.setBody( body )

        registerdata = ModuleDefinition( 'ILDRegisterOutputData' )
        registerdata.setDescription( 'Module to add in the metadata catalog the relevant info about the files' )
        self._addParameter( registerdata, 'enable', 'bool', False, 'EnableFlag' )
        body = string.replace( importLine, '<MODULE>', 'ILDRegisterOutputData' )
        registerdata.setBody( body )

        logUpload = ModuleDefinition( 'UploadLogFile' )
        logUpload.setDescription( 'Uploads the output log files' )
        self._addParameter( logUpload, 'enable', 'bool', False, 'EnableFlag' )
        body = string.replace( importLine, '<MODULE>', 'UploadLogFile' )
        logUpload.setBody( body )

        finalization = StepDefinition( 'Job_Finalization' )
        finalization.addModule( dataUpload )
        up = finalization.createModuleInstance( 'UploadOutputData', 'dataUpload' )
        up.setValue( "enable", self.finalsdict['uploadData'] )

        finalization.addModule( registerdata )
        # TODO: create ILDRegisterOutputData
        ro = finalization.createModuleInstance( 'ILDRegisterOutputData', 'ILDRegisterOutputData' )
        ro.setValue( "enable", self.finalsdict['registerData'] )

        finalization.addModule( logUpload )
        ul = finalization.createModuleInstance( 'UploadLogFile', 'logUpload' )
        ul.setValue( "enable", self.finalsdict['uploadLog'] )

        finalization.addModule( failoverRequest )
        fr = finalization.createModuleInstance( 'FailoverRequest', 'failoverRequest' )
        fr.setValue( "enable", self.finalsdict['sendFailover'] )
        
        self.workflow.addStep( finalization )
        self.workflow.createStepInstance( 'Job_Finalization', 'finalization' )

        return S_OK() 
Esempio n. 19
0
  def addFinalization(self, uploadData=False, registerData=False, uploadLog = False, sendFailover=False):
    """ Add finalization step

    @param uploadData: Upload or not the data to the storage
    @param uploadLog: Upload log file to storage (currently only available for admins, thus add them to OutputSandbox)
    @param sendFailover: Send Failover requests, and declare files as processed or unused in transfDB
    @param registerData: Register data in the file catalog
    @todo: Do the registration only once, instead of once for each job

    """
    self.importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
    dataUpload = ModuleDefinition('UploadOutputData')
    dataUpload.setDescription('Uploads the output data')
    self._addParameter(dataUpload,'enable','bool',False,'EnableFlag')
    body = string.replace(self.importLine,'<MODULE>','UploadOutputData')
    dataUpload.setBody(body)

    failoverRequest = ModuleDefinition('FailoverRequest')
    failoverRequest.setDescription('Sends any failover requests')
    self._addParameter(failoverRequest,'enable','bool',False,'EnableFlag')
    body = string.replace(self.importLine,'<MODULE>','FailoverRequest')
    failoverRequest.setBody(body)

    registerdata = ModuleDefinition('DBDGenRegisterOutputData')
    registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files')
    self._addParameter(registerdata,'enable','bool',False,'EnableFlag')
    body = string.replace(self.importLine,'<MODULE>','DBDGenRegisterOutputData')
    registerdata.setBody(body)

    logUpload = ModuleDefinition('UploadLogFile')
    logUpload.setDescription('Uploads the output log files')
    self._addParameter(logUpload,'enable','bool',False,'EnableFlag')
    body = string.replace(self.importLine,'<MODULE>','UploadLogFile')
    logUpload.setBody(body)

    finalization = StepDefinition('Job_Finalization')
    finalization.addModule(dataUpload)
    up = finalization.createModuleInstance('UploadOutputData','dataUpload')
    up.setValue("enable",uploadData)

    finalization.addModule(registerdata)
    ro = finalization.createModuleInstance('DBDGenRegisterOutputData','DBDGenRegisterOutputData')
    ro.setValue("enable",registerData)

    finalization.addModule(logUpload)
    ul  = finalization.createModuleInstance('UploadLogFile','logUpload')
    ul.setValue("enable",uploadLog)

    finalization.addModule(failoverRequest)
    fr = finalization.createModuleInstance('FailoverRequest','failoverRequest')
    fr.setValue("enable",sendFailover)
    
    self.workflow.addStep(finalization)
    self.workflow.createStepInstance('Job_Finalization', 'finalization')

    return S_OK()
Esempio n. 20
0
File: Job.py Progetto: mesillo/DIRAC
 def __getScriptStep( self, name = 'Script' ):
   """Internal function. This method controls the definition for a script module.
   """
   # Create the script module first
   moduleName = 'Script'
   module = ModuleDefinition( moduleName )
   module.setDescription( 'A  script module that can execute any provided script.' )
   body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n'
   module.setBody( body )
   # Create Step definition
   step = StepDefinition( name )
   step.addModule( module )
   moduleInstance = step.createModuleInstance( 'Script', name )
   # Define step parameters
   step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) )
   step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) )
   step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) )
   step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) )
   return step
Esempio n. 21
0
  def _addRealFinalization(self):  
    """ This is called at creation: now that the workflow is created at the last minute,
    we need to add this also at the last minute
    """
    importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>'
    
    dataUpload = ModuleDefinition('UploadOutputData')
    dataUpload.setDescription('Uploads the output data')
    self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag')
    body = importLine.replace('<MODULE>', 'UploadOutputData')
    dataUpload.setBody(body)

    failoverRequest = ModuleDefinition('FailoverRequest')
    failoverRequest.setDescription('Sends any failover requests')
    self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag')
    body = importLine.replace('<MODULE>', 'FailoverRequest')
    failoverRequest.setBody(body)

    registerdata = ModuleDefinition('RegisterOutputData')
    registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files')
    self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag')
    body = importLine.replace('<MODULE>', 'RegisterOutputData')
    registerdata.setBody(body)

    logUpload = ModuleDefinition('UploadLogFile')
    logUpload.setDescription('Uploads the output log files')
    self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag')
    body = importLine.replace('<MODULE>', 'UploadLogFile')
    logUpload.setBody(body)

    errorReport = ModuleDefinition('ReportErrors')
    errorReport.setDescription('Reports errors at the end')
    body = importLine.replace('<MODULE>', 'ReportErrors')
    errorReport.setBody(body)

    finalization = StepDefinition('Job_Finalization')
    finalization.addModule(dataUpload)
    up = finalization.createModuleInstance('UploadOutputData', 'dataUpload')
    up.setValue("enable", self.finalsdict['uploadData'])

    finalization.addModule(registerdata)
    ro = finalization.createModuleInstance('RegisterOutputData', 'RegisterOutputData')
    ro.setValue("enable", self.finalsdict['registerData'])

    finalization.addModule(logUpload)
    ul  = finalization.createModuleInstance('UploadLogFile', 'logUpload')
    ul.setValue("enable", self.finalsdict['uploadLog'])

    finalization.addModule(failoverRequest)
    fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest')
    fr.setValue("enable", self.finalsdict['sendFailover'])

    finalization.addModule(errorReport)
    fr = finalization.createModuleInstance('ReportErrors', 'reportErrors')

    self.workflow.addStep(finalization)
    self.workflow.createStepInstance('Job_Finalization', 'finalization')

    return S_OK()