コード例 #1
0
ファイル: Test_Utilities.py プロジェクト: TaykYoku/DIRAC
    def test__getStepDefinition(self):
        importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
        # modules
        gaudiApp = ModuleDefinition("Script")
        body = importLine.replace("<MODULE>", "Script")
        gaudiApp.setDescription(
            getattr(
                __import__("%s.%s" % ("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), "__doc__"
            )
        )
        gaudiApp.setBody(body)

        genBKReport = ModuleDefinition("FailoverRequest")
        body = importLine.replace("<MODULE>", "FailoverRequest")
        genBKReport.setDescription(
            getattr(
                __import__("%s.%s" % ("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"]),
                "__doc__",
            )
        )
        genBKReport.setBody(body)

        # step
        appDefn = StepDefinition("App_Step")
        appDefn.addModule(gaudiApp)
        appDefn.createModuleInstance("Script", "Script")
        appDefn.addModule(genBKReport)
        appDefn.createModuleInstance("FailoverRequest", "FailoverRequest")

        appDefn.addParameterLinked(gaudiApp.parameters)

        stepDef = getStepDefinition("App_Step", ["Script", "FailoverRequest"])

        self.assertTrue(str(appDefn) == str(stepDef))

        self.job._addParameter(appDefn, "name", "type", "value", "desc")
        self.job._addParameter(appDefn, "name1", "type1", "value1", "desc1")

        stepDef = getStepDefinition(
            "App_Step",
            ["Script", "FailoverRequest"],
            parametersList=[["name", "type", "value", "desc"], ["name1", "type1", "value1", "desc1"]],
        )

        self.assertTrue(str(appDefn) == str(stepDef))
コード例 #2
0
    def test__getStepDefinition(self):
        importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
        # modules
        gaudiApp = ModuleDefinition('Script')
        body = importLine.replace('<MODULE>', 'Script')
        gaudiApp.setDescription(
            getattr(
                __import__("%s.%s" % ('DIRAC.Workflow.Modules', 'Script'),
                           globals(), locals(), ['__doc__']), "__doc__"))
        gaudiApp.setBody(body)

        genBKReport = ModuleDefinition('FailoverRequest')
        body = importLine.replace('<MODULE>', 'FailoverRequest')
        genBKReport.setDescription(
            getattr(
                __import__(
                    "%s.%s" % ('DIRAC.Workflow.Modules', 'FailoverRequest'),
                    globals(), locals(), ['__doc__']), "__doc__"))
        genBKReport.setBody(body)

        # step
        appDefn = StepDefinition('App_Step')
        appDefn.addModule(gaudiApp)
        appDefn.createModuleInstance('Script', 'Script')
        appDefn.addModule(genBKReport)
        appDefn.createModuleInstance('FailoverRequest', 'FailoverRequest')

        appDefn.addParameterLinked(gaudiApp.parameters)

        stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'])

        self.assert_(str(appDefn) == str(stepDef))

        self.job._addParameter(appDefn, 'name', 'type', 'value', 'desc')
        self.job._addParameter(appDefn, 'name1', 'type1', 'value1', 'desc1')

        stepDef = getStepDefinition(
            'App_Step', ['Script', 'FailoverRequest'],
            parametersList=[['name', 'type', 'value', 'desc'],
                            ['name1', 'type1', 'value1', 'desc1']])

        self.assert_(str(appDefn) == str(stepDef))
コード例 #3
0
ファイル: Test_Utilities.py プロジェクト: DIRACGrid/DIRAC
  def test__getStepDefinition( self ):
    importLine = """
from DIRAC.Workflow.Modules.<MODULE> import <MODULE>
"""
    # modules
    gaudiApp = ModuleDefinition( 'Script' )
    body = importLine.replace( '<MODULE>', 'Script' )
    gaudiApp.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'Script' ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
    gaudiApp.setBody( body )

    genBKReport = ModuleDefinition( 'FailoverRequest' )
    body = importLine.replace( '<MODULE>', 'FailoverRequest' )
    genBKReport.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'FailoverRequest' ),
                                                     globals(), locals(), ['__doc__'] ),
                                        "__doc__" ) )
    genBKReport.setBody( body )

    # step
    appDefn = StepDefinition( 'App_Step' )
    appDefn.addModule( gaudiApp )
    appDefn.createModuleInstance( 'Script', 'Script' )
    appDefn.addModule( genBKReport )
    appDefn.createModuleInstance( 'FailoverRequest', 'FailoverRequest' )

    appDefn.addParameterLinked( gaudiApp.parameters )

    stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'] )

    self.assertTrue( str( appDefn ) == str( stepDef ) )



    self.job._addParameter( appDefn, 'name', 'type', 'value', 'desc' )
    self.job._addParameter( appDefn, 'name1', 'type1', 'value1', 'desc1' )


    stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'],
                                 parametersList = [[ 'name', 'type', 'value', 'desc' ],
                                                   [ 'name1', 'type1', 'value1', 'desc1' ]] )

    self.assertTrue( str( appDefn ) == str( stepDef ) )
コード例 #4
0
ファイル: Production.py プロジェクト: antolu/LHCbDIRAC
    def addFinalizationStep(self, modulesList=None):
        """ Add the finalization step to the workflow (some defaults are inserted)

        :param list modulesList: the list of modules names this step is made of. If None, a default is taken.
        :returns: None
    """
        if modulesList is None:
            modulesList = [
                'UploadOutputData', 'UploadLogFile', 'UploadMC',
                'FailoverRequest'
            ]

        if 'Job_Finalization' not in self.LHCbJob.workflow.step_definitions.keys(
        ):

            jobFinalizationStepDef = getStepDefinition(
                'Job_Finalization',
                importLine='LHCbDIRAC.Workflow.Modules',
                modulesNameList=modulesList)
            self.LHCbJob.workflow.addStep(jobFinalizationStepDef)

        # create the step instance add it to the wf
        self.LHCbJob.workflow.createStepInstance('Job_Finalization',
                                                 'finalization')
コード例 #5
0
ファイル: Job.py プロジェクト: graciani/DIRAC
  def setExecutable( self, executable, arguments = '', logFile = '',
                       modulesList = ['Script'],
                       parameters = [( 'executable', 'string', '', "Executable Script" ),
                                     ( 'arguments', 'string', '', 'Arguments for executable Script' ),
                                     ( 'applicationLog', 'string', '', "Log file name" )],
                       paramValues = [] ):
    """Helper function.

       Specify executable script to run with optional arguments and log file
       for standard output.

       These can be either:

        - Submission of a python or shell script to DIRAC
           - Can be inline scripts e.g. C{'/bin/ls'}
           - Scripts as executables e.g. python or shell script file

       Example usage:

       >>> job = Job()
       >>> job.setExecutable('myScript.py')

       @param executable: Executable
       @type executable: string
       @param arguments: Optional arguments to executable
       @type arguments: string
       @param logFile: Optional log file name
       @type logFile: string
       @param modulesList: Optional list of modules (to be used mostly when extending this method)
       @type modulesList: list
       @param parameters: Optional list of parameters (to be used mostly when extending this method)
       @type parameters: list of tuples
       @param paramValues: Optional list of parameters values (to be used mostly when extending this method)
       @type parameters: list of tuples
    """
    kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
    if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or \
       not type( logFile ) == type( ' ' ):
      return self._reportError( 'Expected strings for executable and arguments', **kwargs )

    if os.path.exists( executable ):
      self.log.verbose( 'Found script executable file %s' % ( executable ) )
      self.addToInputSandbox.append( executable )
      logName = '%s.log' % ( os.path.basename( executable ) )
    else:
      self.log.warn( 'The executable code could not be found locally' )
      logName = 'CodeOutput.log'

    if logFile:
      if type( logFile ) == type( ' ' ):
        logName = str(logFile)

    self.stepCount += 1
    stepName = 'RunScriptStep%s' % ( self.stepCount )

    step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
    self.addToOutputSandbox.append( logName )

    stepInstance = addStepToWorkflow( self.workflow, step, stepName )

    stepInstance.setValue( 'applicationLog', logName )
    stepInstance.setValue( 'executable', executable )
    if arguments:
      stepInstance.setValue( 'arguments', arguments )
    if paramValues:
      for param, value in paramValues:
        stepInstance.setValue( param, value )

    return S_OK( stepInstance )
コード例 #6
0
ファイル: LHCbJob.py プロジェクト: antolu/LHCbDIRAC
    def setExecutable(self,
                      executable,
                      arguments=None,
                      logFile=None,
                      systemConfig='ANY',
                      modulesNameList=None,
                      parametersList=None):
        """Specifies executable script to run with optional arguments and log file
       for standard output.

       These can be either:

        - Submission of a python or shell script to DIRAC
           - Can be inline scripts e.g. C{'/bin/ls'}
           - Scripts as executables e.g. python or shell script file

       Example usage:

       >>> job = Job()
       >>> job.setExecutable('myScript.py')

       :param executable: Executable
       :type executable: string
       :param arguments: Optional arguments to executable
       :type arguments: string
       :param logFile: Optional log file name
       :type logFile: string
    """
        kwargs = {
            'executable': executable,
            'arguments': arguments,
            'applicationLog': logFile
        }
        if not isinstance(executable, str):
            return self._reportError(
                'Expected strings for executable and arguments', **kwargs)

        if os.path.exists(executable):
            self.log.verbose('Found script executable file %s' % (executable))
            self.addToInputSandbox.append(executable)
            logName = '%s.log' % (os.path.basename(executable))
            moduleName = os.path.basename(executable)
        else:
            self.log.verbose('Found executable code')
            logName = 'CodeOutput.log'
            moduleName = 'CodeSegment'

        if logFile:
            if isinstance(logFile, str):
                logName = str(logFile)

        self.stepCount += 1

        moduleName = moduleName.replace('.', '')
        stepName = 'ScriptStep%s' % (self.stepCount)

        if not modulesNameList:
            modulesNameList = [
                'CreateDataFile', 'LHCbScript', 'FileUsage',
                'UserJobFinalization'
            ]
        if not parametersList:
            parametersList = [
                ('name', 'string', '', 'Name of executable'),
                ('executable', 'string', '', 'Executable Script'),
                ('arguments', 'string', '', 'Arguments for executable Script'),
                ('applicationLog', 'string', '', 'Log file name'),
                ('SystemConfig', 'string', '', 'binary tag')
            ]

        step = getStepDefinition(stepName,
                                 modulesNameList=modulesNameList,
                                 importLine="LHCbDIRAC.Workflow.Modules",
                                 parametersList=parametersList)

        stepName = 'RunScriptStep%s' % (self.stepCount)
        logPrefix = 'Script%s_' % (self.stepCount)
        logName = '%s%s' % (logPrefix, logName)
        self.addToOutputSandbox.append(logName)

        stepInstance = addStepToWorkflow(self.workflow, step, stepName)

        stepInstance.setValue('name', moduleName)
        stepInstance.setValue('applicationLog', logName)
        stepInstance.setValue('executable', executable)
        stepInstance.setValue('SystemConfig', systemConfig)
        if arguments:
            stepInstance.setValue('arguments', arguments)

        return S_OK()
コード例 #7
0
ファイル: LHCbJob.py プロジェクト: antolu/LHCbDIRAC
    def __configureRootModule(self, rootVersion, rootScript, rootType,
                              arguments, logFile, systemConfig):
        """ Internal function.

        Supports the root macro, python and executable wrapper functions.
    """
        kwargs = {
            'rootVersion': rootVersion,
            'rootScript': rootScript,
            'rootType': rootType,
            'arguments': arguments,
            'logFile': logFile
        }
        if not isinstance(arguments, list):
            arguments = [arguments]

        for param in [rootVersion, rootScript, rootType, logFile]:
            if not isinstance(param, str):
                return self._reportError(
                    'Expected strings for Root application input parameters',
                    __name__, **kwargs)

        if not os.path.exists(rootScript):
            return self._reportError(
                'ROOT Script %s must exist locally' % (rootScript), __name__,
                **kwargs)

        self.addToInputSandbox.append(rootScript)

        rootName = os.path.basename(rootScript).replace('.', '')
        if logFile:
            logName = logFile
        else:
            logName = '%s_%s.log' % (rootName, rootVersion.replace('.', ''))

        self.stepCount += 1
        stepName = '%sStep%s' % (rootName, self.stepCount)

        modulesNameList = [
            'CreateDataFile', 'RootApplication', 'FileUsage',
            'UserJobFinalization'
        ]
        parametersList = [('rootVersion', 'string', '', 'Root version'),
                          ('rootScript', 'string', '', 'Root script'),
                          ('rootType', 'string', '', 'Root type'),
                          ('arguments', 'list', [],
                           'Optional arguments for payload'),
                          ('applicationLog', 'string', '', 'Log file name'),
                          ('SystemConfig', 'string', '', 'binary tag')]
        step = getStepDefinition(stepName,
                                 modulesNameList=modulesNameList,
                                 importLine="LHCbDIRAC.Workflow.Modules",
                                 parametersList=parametersList)

        logPrefix = 'Step%s_' % (self.stepCount)
        logName = '%s%s' % (logPrefix, logName)
        self.addToOutputSandbox.append(logName)

        stepInstance = addStepToWorkflow(self.workflow, step, stepName)

        # Define Step and its variables
        stepInstance.setValue("rootVersion", rootVersion)
        stepInstance.setValue("rootType", rootType)
        stepInstance.setValue("rootScript", os.path.basename(rootScript))
        stepInstance.setValue("applicationLog", logName)
        stepInstance.setValue("SystemConfig", systemConfig)
        if arguments:
            stepInstance.setValue("arguments", arguments)

        return S_OK(stepInstance)
コード例 #8
0
ファイル: LHCbJob.py プロジェクト: antolu/LHCbDIRAC
    def setApplicationScript(self,
                             appName,
                             appVersion,
                             script,
                             arguments='',
                             inputData='',
                             inputDataType='',
                             poolXMLCatalog='pool_xml_catalog.xml',
                             logFile='',
                             extraPackages='',
                             systemConfig='ANY',
                             modulesNameList=None,
                             parametersList=None):
        """Specifies application environment and script to be executed.

       For LHCb these could be e.g. Gauss, Boole, Brunel,
       DaVinci etc.

       The script name and any arguments should also be specified.

       Input data for application script must be specified here, please note that if this is set at the job level,
       e.g. via setInputData() but not above, input data is not in the scope of the specified application.

       Any input data specified at the step level that is not already specified at the job level is added automatically
       as a requirement for the job.

       Example usage:

       >>> job = LHCbJob()
       >>> job.setApplicationScript('DaVinci','v19r12','myScript.py')

       :param appName: Application name
       :type appName: string
       :param appVersion: Application version
       :type appVersion: string
       :param script: Script to execute
       :type script: string
       :param arguments: Optional arguments for script
       :type arguments: string
       :param inputData: Input data for application
       :type inputData: single LFN or list of LFNs
       :param inputDataType: Input data type for application (e.g. DATA, MDF, ETC)
       :type inputDataType: string
       :param arguments: Optional POOL XML Catalog name for any input data files (default is pool_xml_catalog.xml)
       :type arguments: string
       :param logFile: Optional log file name
       :type logFile: string
       :param modulesNameList: list of module names. The default is given.
       :type modulesNameList: list
       :param parametersList: list of parameters. The default is given.
       :type parametersList: list
    """
        kwargs = {
            'appName': appName,
            'appVersion': appVersion,
            'script': script,
            'arguments': arguments,
            'inputData': inputData,
            'inputDataType': inputDataType,
            'poolXMLCatalog': poolXMLCatalog,
            'logFile': logFile
        }
        if not isinstance(appName, str) or not isinstance(appVersion, str):
            return self._reportError(
                'Expected strings for application name and version', __name__,
                **kwargs)

        if not script or not isinstance(script, str):
            return self._reportError('Expected strings for script name',
                                     __name__, **kwargs)

        if not os.path.exists(script):
            return self._reportError('Script must exist locally', __name__,
                                     **kwargs)

        if logFile:
            if isinstance(logFile, str):
                logName = logFile
            else:
                return self._reportError('Expected string for log file name',
                                         __name__, **kwargs)
        else:
            shortScriptName = os.path.basename(script).split('.')[0]
            logName = '%s_%s_%s.log' % (appName, appVersion, shortScriptName)

        self.addToInputSandbox.append(script)

        if arguments:
            if not isinstance(arguments, str):
                return self._reportError(
                    'Expected string for optional script arguments', __name__,
                    **kwargs)

        if not isinstance(poolXMLCatalog, str):
            return self._reportError(
                'Expected string for POOL XML Catalog name', __name__,
                **kwargs)

        if inputData:
            if isinstance(inputData, str):
                inputData = [inputData]
            if not isinstance(inputData, list):
                return self._reportError(
                    'Expected single LFN string or list of LFN(s) for inputData',
                    __name__, **kwargs)
            if inputData != ['previousStep']:
                for i in xrange(len(inputData)):
                    inputData[i] = inputData[i].replace('LFN:', '')
                inputData = ['LFN:' + x for x in inputData]
                inputDataStr = ';'.join(inputData)
                self.addToInputData.append(inputDataStr)

        self.stepCount += 1

        stepName = '%sStep%s' % (appName, self.stepCount)

        if not modulesNameList:
            modulesNameList = [
                'CreateDataFile', 'GaudiApplicationScript', 'FileUsage',
                'UserJobFinalization'
            ]
        if not parametersList:
            parametersList = [
                ('applicationName', 'string', '', 'Application Name'),
                ('applicationVersion', 'string', '', 'Application Version'),
                ('applicationLog', 'string', '',
                 'Name of the output file of the application'),
                ('script', 'string', '', 'Script Name'),
                ('arguments', 'string', '', 'Arguments for executable'),
                ('poolXMLCatName', 'string', '', 'POOL XML Catalog file name'),
                ('inputDataType', 'string', '', 'Input Data Type'),
                ('inputData', 'string', '', 'Input Data'),
                ('extraPackages', 'string', '', 'extraPackages'),
                ('SystemConfig', 'string', '', 'binary tag')
            ]

        step = getStepDefinition(stepName,
                                 modulesNameList=modulesNameList,
                                 importLine="LHCbDIRAC.Workflow.Modules",
                                 parametersList=parametersList)

        logPrefix = 'Step%s_' % (self.stepCount)
        logName = '%s%s' % (logPrefix, logName)
        self.addToOutputSandbox.append(logName)

        stepInstance = addStepToWorkflow(self.workflow, step, stepName)

        stepInstance.setValue("applicationName", appName)
        stepInstance.setValue("applicationVersion", appVersion)
        stepInstance.setValue("script", script)
        stepInstance.setValue("applicationLog", logName)
        if arguments:
            stepInstance.setValue("arguments", arguments)
        if inputDataType:
            stepInstance.setValue("inputDataType", inputDataType)
        if inputData:
            stepInstance.setValue("inputData", ';'.join(inputData))
        if poolXMLCatalog:
            stepInstance.setValue("poolXMLCatName", poolXMLCatalog)
        stepInstance.setValue('extraPackages', extraPackages)
        stepInstance.setValue('SystemConfig', systemConfig)

        return S_OK(stepInstance)
コード例 #9
0
ファイル: LHCbJob.py プロジェクト: antolu/LHCbDIRAC
    def setApplication(self,
                       appName,
                       appVersion,
                       optionsFiles,
                       inputData=None,
                       optionsLine=None,
                       inputDataType=None,
                       logFile=None,
                       events=-1,
                       extraPackages=None,
                       systemConfig='ANY',
                       modulesNameList=None,
                       parametersList=None):
        """Specifies gaudi application for DIRAC workflows, executed via gaudirun.

       For LHCb these could be e.g. Gauss, Boole, Brunel, DaVinci, LHCb, etc.

       The optionsFiles parameter can be the path to an options file or a list of paths to options files.
       All options files are automatically appended to the job input sandBox but the first in the case of a
       list is assumed to be the 'master' options file.

       Input data for application script must be specified here, please note that if this is set at the job level,
       e.g. via setInputData() but not above, input data is not in the scope of the specified application.

       Any input data specified at the step level that is not already specified at the job level is added automatically
       as a requirement for the job.

       Example usage:

       >>> job = LHCbJob()
       >>> job.setApplication('DaVinci','v19r5',optionsFiles='MyDV.opts',
       inputData=['/lhcb/production/DC06/phys-lumi2/00001501/DST/0000/00001501_00000320_5.dst'],logFile='dv.log')

       :param appName: Application name
       :type appName: string
       :param appVersion: Application version
       :type appVersion: string
       :param optionsFiles: Path to options file(s) for application
       :type optionsFiles: string or list
       :param inputData: Input data for application (if a subset of the overall input data for a given job is required)
       :type inputData: single LFN or list of LFNs
       :param optionsLine: Additional options lines for application
       :type optionsLine: string
       :param inputDataType: Input data type for application (e.g. DATA, MDF, ETC)
       :type inputDataType: string
       :param logFile: Optional log file name
       :type logFile: string
       :param events: Optional number of events
       :type events: integer
       :param extraPackages: Optional extra packages
       :type extraPackages: string
       :param modulesNameList: list of module names. The default is given.
       :type modulesNameList: list
       :param parametersList: list of parameters. The default is given.
       :type parametersList: list
    """
        kwargs = {
            'appName': appName,
            'appVersion': appVersion,
            'optionsFiles': optionsFiles,
            'inputData': inputData,
            'optionsLine': optionsLine,
            'inputDataType': inputDataType,
            'logFile': logFile
        }
        if not isinstance(appName, str) or not isinstance(appVersion, str):
            return self._reportError(
                'Expected strings for application name and version', __name__,
                **kwargs)

        if logFile:
            if isinstance(logFile, str):
                logName = logFile
            else:
                return self._reportError('Expected string for log file name',
                                         __name__, **kwargs)
        else:
            logName = '%s_%s.log' % (appName, appVersion)

        if not inputDataType:
            inputDataType = self.inputDataType
        if not isinstance(inputDataType, str):
            return self._reportError('Expected string for input data type',
                                     __name__, **kwargs)

        optionsFile = None
        if not optionsFiles:
            return self._reportError(
                'Expected string or list for optionsFiles', __name__, **kwargs)
        if isinstance(optionsFiles, str):
            optionsFiles = [optionsFiles]
        if not isinstance(optionsFiles, list):
            return self._reportError(
                'Expected string or list for optionsFiles', __name__, **kwargs)
        for optsFile in optionsFiles:
            if not optionsFile:
                self.log.verbose('Found master options file %s' % optsFile)
                optionsFile = optsFile
            if os.path.exists(optsFile):
                self.log.verbose('Found specified options file: %s' % optsFile)
                self.addToInputSandbox.append(optsFile)
                optionsFile += ';%s' % optsFile
            elif re.search(r'\$', optsFile):
                self.log.verbose(
                    'Assuming %s is using an environment variable to be resolved during execution'
                    % optsFile)
                if not optionsFile == optsFile:
                    optionsFile += ';%s' % optsFile
            else:
                return self._reportError(
                    'Specified options file %s does not exist' % (optsFile),
                    __name__, **kwargs)

        # ensure optionsFile list is unique:
        tmpList = optionsFile.split(';')
        optionsFile = ';'.join(list(set(tmpList)))
        self.log.verbose('Final options list is: %s' % optionsFile)

        if inputData:
            if isinstance(inputData, str):
                inputData = [inputData]
            if not isinstance(inputData, list):
                return self._reportError(
                    'Expected single LFN string or list of LFN(s) for inputData',
                    __name__, **kwargs)
            if inputData != ['previousStep']:
                for i in xrange(len(inputData)):
                    inputData[i] = inputData[i].replace('LFN:', '')
                inputData = ['LFN:' + x for x in inputData]
                inputDataStr = ';'.join(inputData)
                self.addToInputData.append(inputDataStr)

        self.stepCount += 1
        stepName = '%sStep%s' % (appName, self.stepCount)

        if not modulesNameList:
            modulesNameList = [
                'CreateDataFile', 'GaudiApplication', 'FileUsage',
                'AnalyseFileAccess', 'UserJobFinalization'
            ]
        if not parametersList:
            parametersList = [
                ('applicationName', 'string', '', 'Application Name'),
                ('applicationVersion', 'string', '', 'Application Version'),
                ('applicationLog', 'string', '',
                 'Name of the output file of the application'),
                ('optionsFile', 'string', '', 'Options File'),
                ('extraOptionsLine', 'string', '',
                 'This is appended to standard options'),
                ('inputDataType', 'string', '', 'Input Data Type'),
                ('inputData', 'string', '', 'Input Data'),
                ('numberOfEvents', 'string', '', 'Events treated'),
                ('extraPackages', 'string', '', 'ExtraPackages'),
                ('SystemConfig', 'string', '', 'binary tag')
            ]

        step = getStepDefinition(stepName,
                                 modulesNameList=modulesNameList,
                                 importLine="LHCbDIRAC.Workflow.Modules",
                                 parametersList=parametersList)

        logPrefix = 'Step%s_' % (self.stepCount)
        logName = '%s%s' % (logPrefix, logName)
        self.addToOutputSandbox.append(logName)

        stepInstance = addStepToWorkflow(self.workflow, step, stepName)

        stepInstance.setValue('applicationName', appName)
        stepInstance.setValue('applicationVersion', appVersion)
        stepInstance.setValue('applicationLog', logName)
        if optionsFile:
            stepInstance.setValue('optionsFile', optionsFile)
        if optionsLine:
            stepInstance.setValue('extraOptionsLine', optionsLine)
        if inputDataType:
            stepInstance.setValue('inputDataType', inputDataType)
        if inputData:
            stepInstance.setValue('inputData', ';'.join(inputData))
        stepInstance.setValue('numberOfEvents', str(events))
        stepInstance.setValue('extraPackages', extraPackages)
        stepInstance.setValue('SystemConfig', systemConfig)

        return S_OK(stepInstance)
コード例 #10
0
ファイル: Job.py プロジェクト: ahaupt/DIRAC
  def setExecutable( self, executable, arguments = '', logFile = '',
                     modulesList = None, parameters = None, paramValues = None ):
    """Helper function.

       Specify executable script to run with optional arguments and log file
       for standard output.

       These can be either:

       - Submission of a python or shell script to DIRAC
          - Can be inline scripts e.g. C{'/bin/ls'}
          - Scripts as executables e.g. python or shell script file

       Example usage:

       >>> job = Job()
       >>> job.setExecutable('myScript.py')

       :param executable: Executable
       :type executable: string
       :param arguments: Optional arguments to executable
       :type arguments: string
       :param logFile: Optional log file name
       :type logFile: string
       :param modulesList: Optional list of modules (to be used mostly when extending this method)
       :type modulesList: list
       :param parameters: Optional list of parameters (to be used mostly when extending this method)
       :type parameters: list of tuples
       :param paramValues: Optional list of parameters values (to be used mostly when extending this method)
       :type parameters: list of tuples
    """
    kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
    if not isinstance( executable, basestring ) or not isinstance( arguments, basestring ) or \
       not isinstance( logFile, basestring ):
      return self._reportError( 'Expected strings for executable and arguments', **kwargs )

    if os.path.exists( executable ):
      self.log.verbose( 'Found script executable file %s' % ( executable ) )
      self.addToInputSandbox.append( executable )
      logName = '%s.log' % ( os.path.basename( executable ) )
    else:
      self.log.warn( 'The executable code could not be found locally' )
      logName = 'CodeOutput.log'

    self.stepCount += 1
    stepName = 'RunScriptStep%s' % ( self.stepCount )

    if logFile:
      if isinstance( logFile, basestring ):
        logName = str(logFile)
    else:
      logName = "Script%s_%s" %( self.stepCount, logName )

    if not modulesList:
      modulesList = ['Script']
    if not parameters:
      parameters = [( 'executable', 'string', '', "Executable Script" ),
                    ( 'arguments', 'string', '', 'Arguments for executable Script' ),
                    ( 'applicationLog', 'string', '', "Log file name" )]

    step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
    self.addToOutputSandbox.append( logName )

    stepInstance = addStepToWorkflow( self.workflow, step, stepName )

    stepInstance.setValue( 'applicationLog', logName )
    stepInstance.setValue( 'executable', executable )
    if arguments:
      # If arguments are expressed in terms of parameters, pass them to Workflow
      # These arguments will be resolved on the server side for each parametric job
      if re.search( '%\(.*\)s', arguments ) or re.search( '%n', arguments ):
        self.parametricWFArguments['arguments'] = arguments
      else:
        stepInstance.setValue( 'arguments', arguments )
    if paramValues:
      for param, value in paramValues:
        stepInstance.setValue( param, value )

    return S_OK( stepInstance )
コード例 #11
0
ファイル: Job.py プロジェクト: ptakha/DIRAC-1
  def setExecutable( self, executable, arguments = '', logFile = '',
                       modulesList = ['Script'],
                       parameters = [( 'executable', 'string', '', "Executable Script" ),
                                     ( 'arguments', 'string', '', 'Arguments for executable Script' ),
                                     ( 'applicationLog', 'string', '', "Log file name" )],
                       paramValues = [] ):
    """Helper function.

       Specify executable script to run with optional arguments and log file
       for standard output.

       These can be either:

        - Submission of a python or shell script to DIRAC
           - Can be inline scripts e.g. C{'/bin/ls'}
           - Scripts as executables e.g. python or shell script file

       Example usage:

       >>> job = Job()
       >>> job.setExecutable('myScript.py')

       @param executable: Executable
       @type executable: string
       @param arguments: Optional arguments to executable
       @type arguments: string
       @param logFile: Optional log file name
       @type logFile: string
       @param modulesList: Optional list of modules (to be used mostly when extending this method)
       @type modulesList: list
       @param parameters: Optional list of parameters (to be used mostly when extending this method)
       @type parameters: list of tuples
       @param paramValues: Optional list of parameters values (to be used mostly when extending this method)
       @type parameters: list of tuples
    """
    kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile}
    if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or \
       not type( logFile ) == type( ' ' ):
      return self._reportError( 'Expected strings for executable and arguments', **kwargs )

    if os.path.exists( executable ):
      self.log.verbose( 'Found script executable file %s' % ( executable ) )
      self.addToInputSandbox.append( executable )
      logName = '%s.log' % ( os.path.basename( executable ) )
    else:
      self.log.warn( 'The executable code could not be found locally' )
      logName = 'CodeOutput.log'

    self.stepCount += 1
    stepName = 'RunScriptStep%s' % ( self.stepCount )

    if logFile:
      if type( logFile ) == type( ' ' ):
        logName = str(logFile)
    else:
      logName = "Script%s_%s" %( self.stepCount, logName )

    step = getStepDefinition( 'ScriptStep%s' % ( self.stepCount ), modulesList, parametersList = parameters )
    self.addToOutputSandbox.append( logName )

    stepInstance = addStepToWorkflow( self.workflow, step, stepName )

    stepInstance.setValue( 'applicationLog', logName )
    stepInstance.setValue( 'executable', executable )
    if arguments:
      stepInstance.setValue( 'arguments', arguments )
    if paramValues:
      for param, value in paramValues:
        stepInstance.setValue( param, value )

    return S_OK( stepInstance )
コード例 #12
0
ファイル: Production.py プロジェクト: antolu/LHCbDIRAC
    def addApplicationStep(self, stepDict, inputData=None, modulesList=None):
        """ Adds an application step to the workflow

        stepDict contains everything that is in the step, for this production, e.g.::

          {'ApplicationName': 'DaVinci', 'Usable': 'Yes', 'StepId': 13718, 'ApplicationVersion': 'v28r3p1',
          'ExtraPackages': 'AppConfig.v3r104', 'StepName': 'Stripping14-Merging', 'ExtraOptions': '',
          'ProcessingPass': '******', 'Visible': 'N', 'OptionsFormat': '',
          'OptionFiles': '$APPCONFIGOPTS/Merging/DV-Stripping14-Merging.py',
          'DDDB': 'head-20110302', 'CONDDB': 'head-20110407', 'DQTag': '',
          'isMulticore': 'N', 'SystemConfig': '', 'mcTCK': '',
          'fileTypesIn': ['SDST'],
          'visibilityFlag': [{'Visible': 'Y', 'FileType': 'BHADRON.DST'}],
          'fileTypesOut': ['BHADRON.DST', 'CALIBRATION.DST', 'CHARM.MDST', 'CHARMCOMPLETEEVENT.DST']}

        Note: this step treated here does not necessarily corresponds to a step of the BKK:
        the case where they might be different is the merging case.

        :param dict stepDict: contains everything that is in the step, for this production
        :param str inputData: the input data of the step. Either None, or 'previousStep', or the input to the step
        :param list modulesList: the list of module names (str) this step is made of. If None, a default is taken.

        :returns: the name (str) of the step added
    """

        appName = stepDict['ApplicationName']
        appVersion = stepDict['ApplicationVersion']
        optionsFile = stepDict['OptionFiles']
        stepID = stepDict['StepId']
        stepName = stepDict['StepName']
        stepVisible = stepDict['Visible']
        extraPackages = stepDict['ExtraPackages']
        optionsLine = stepDict['ExtraOptions']
        fileTypesIn = stepDict['fileTypesIn']
        fileTypesOut = stepDict['fileTypesOut']
        stepPass = stepDict['ProcessingPass']
        optionsFormat = stepDict['OptionsFormat']
        dddbOpt = stepDict['DDDB']
        conddbOpt = stepDict['CONDDB']
        dqOpt = stepDict['DQTag']
        multicore = stepDict['isMulticore']
        sysConfig = stepDict['SystemConfig']
        outputVisibility = stepDict['visibilityFlag']
        if sysConfig == 'None' or sysConfig == 'NULL' or not sysConfig or sysConfig is None:
            sysConfig = 'ANY'
        mcTCK = stepDict['mcTCK']

        if extraPackages:
            if isinstance(extraPackages, list):
                extraPackages = ';'.join(extraPackages)
            if not isinstance(extraPackages, str):
                raise TypeError("extraPackages is not a string (nor a list)")
            if ',' in extraPackages:
                extraPackages = ';'.join(extraPackages.split(','))
            if 'ProdConf' not in extraPackages:
                extraPackages = extraPackages + ';ProdConf'
            extraPackages = extraPackages.replace(' ', '')
        else:
            extraPackages = 'ProdConf'

        self.__checkArguments(extraPackages, optionsFile)

        try:
            if not dddbOpt.lower() == 'global':
                gLogger.verbose(
                    "Specific DDDBTag setting found for %s step, setting to: %s"
                    % (appName, dddbOpt))
                dddbOpt = dddbOpt.replace(' ', '')
        except AttributeError:
            pass
        try:
            if not conddbOpt.lower() == 'global':
                gLogger.verbose(
                    "Specific CondDBTag setting found for %s step, setting to: %s"
                    % (appName, conddbOpt))
                conddbOpt = conddbOpt.replace(' ', '')
        except AttributeError:
            pass
        try:
            if not dqOpt.lower() == 'global':
                gLogger.verbose(
                    "Specific DQTag setting found for %s step, setting to: %s"
                    % (appName, dqOpt))
                dqOpt = dqOpt.replace(' ', '')
        except AttributeError:
            pass

        # starting real stuff
        self.LHCbJob.stepCount += 1

        if 'Gaudi_App_Step' not in self.LHCbJob.workflow.step_definitions.keys(
        ):

            gLogger.debug(
                "Determining the modules of the steps (modulesList = %s)" %
                modulesList)
            if modulesList is None:  # we assume it's a standard list of modules for Gaudi steps
                gaudiPath = 'Productions/GaudiStep_Modules'
                modulesList = self.opsHelper.getValue(gaudiPath, [
                    'GaudiApplication', 'AnalyseXMLSummary', 'ErrorLogging',
                    'BookkeepingReport', 'StepAccounting'
                ])

            # pName, pType, pValue, pDesc
            parametersList = [
                ['inputData', 'string', '', 'StepInputData'],
                ['inputDataType', 'string', '', 'InputDataType'],
                ['applicationName', 'string', '', 'ApplicationName'],
                ['applicationVersion', 'string', '', 'ApplicationVersion'],
                ['runTimeProjectName', 'string', '', 'runTimeProjectName'],
                [
                    'runTimeProjectVersion', 'string', '',
                    'runTimeProjectVersion'
                ], ['applicationType', 'string', '', 'ApplicationType'],
                ['optionsFile', 'string', '', 'OptionsFile'],
                ['extraOptionsLine', 'string', '', 'extraOptionsLines'],
                ['listoutput', 'list', [], 'StepOutputList'],
                ['extraPackages', 'string', '', 'ExtraPackages'],
                ['BKStepID', 'string', '', 'BKKStepID'],
                ['StepProcPass', 'string', '', 'StepProcessingPass'],
                ['optionsFormat', 'string', '', 'ProdConf configuration'],
                ['CondDBTag', 'string', '', 'ConditionDatabaseTag'],
                ['DDDBTag', 'string', '', 'DetDescTag'],
                ['DQTag', 'string', '', 'DataQualityTag'],
                ['multiCore', 'string', '', 'MultiCore Flag'],
                ['SystemConfig', 'string', '', 'system config'],
                ['mcTCK', 'string', '', 'TCK to be simulated']
            ]

            gaudiStepDef = getStepDefinition(
                'Gaudi_App_Step',
                modulesNameList=modulesList,
                importLine='LHCbDIRAC.Workflow.Modules',
                parametersList=parametersList)
            self.LHCbJob.workflow.addStep(gaudiStepDef)

        # create the step instance add it to the wf, and return it
        name = '%s_%s' % (appName, self.LHCbJob.stepCount)
        gaudiStepInstance = self.LHCbJob.workflow.createStepInstance(
            'Gaudi_App_Step', name)

        valuesToSet = [['applicationName', appName],
                       ['applicationVersion', appVersion],
                       ['optionsFile', optionsFile],
                       ['extraOptionsLine', optionsLine],
                       ['extraPackages', extraPackages],
                       ['BKStepID', str(stepID)], ['StepProcPass', stepPass],
                       ['optionsFormat', optionsFormat],
                       ['CondDBTag', conddbOpt], ['DDDBTag', dddbOpt],
                       ['DQTag', dqOpt], ['multiCore', multicore],
                       ['SystemConfig', sysConfig], ['mcTCK', mcTCK]]

        if fileTypesIn:
            valuesToSet.append([
                'inputDataType', ';'.join(ftIn.upper() for ftIn in fileTypesIn)
            ])

        if inputData is None:
            gLogger.verbose(
                '%s step has no data requirement or is linked to the overall input data'
                % appName)
            gaudiStepInstance.setLink('inputData', 'self', 'InputData')
        elif inputData == 'previousStep':
            gLogger.verbose(
                'Taking input data as output from previous Gaudi step')
            valuesToSet.append(['inputData', inputData])
        else:
            gLogger.verbose(
                'Assume input data requirement should be added to job')
            self.LHCbJob.setInputData(inputData)
            gaudiStepInstance.setLink('inputData', 'self', 'InputData')

        for pName, value in valuesToSet:
            if value:
                gaudiStepInstance.setValue(pName, value)

        outputFilesList = self._constructOutputFilesList(fileTypesOut)
        gaudiStepInstance.setValue('listoutput', (outputFilesList))

        # to construct the BK processing pass structure, starts from step '0'
        stepIDInternal = 'Step%s' % (self.LHCbJob.stepCount - 1)
        bkOptionsFile = optionsFile

        stepBKInfo = {
            'ApplicationName': appName,
            'ApplicationVersion': appVersion,
            'OptionFiles': bkOptionsFile,
            'DDDb': dddbOpt,
            'CondDb': conddbOpt,
            'DQTag': dqOpt,
            'ExtraPackages': extraPackages,
            'BKStepID': stepID,
            'StepName': stepName,
            'StepVisible': stepVisible,
            'OutputFileTypes': outputVisibility
        }

        self.bkSteps[stepIDInternal] = stepBKInfo
        self.__addBKPassStep()

        return name
コード例 #13
0
ファイル: Job.py プロジェクト: petricm/DIRAC
    def setExecutable(self, executable, arguments="", logFile="", modulesList=None, parameters=None, paramValues=None):
        """Helper function.

       Specify executable script to run with optional arguments and log file
       for standard output.

       These can be either:

        - Submission of a python or shell script to DIRAC
           - Can be inline scripts e.g. C{'/bin/ls'}
           - Scripts as executables e.g. python or shell script file

       Example usage:

       >>> job = Job()
       >>> job.setExecutable('myScript.py')

       @param executable: Executable
       @type executable: string
       @param arguments: Optional arguments to executable
       @type arguments: string
       @param logFile: Optional log file name
       @type logFile: string
       @param modulesList: Optional list of modules (to be used mostly when extending this method)
       @type modulesList: list
       @param parameters: Optional list of parameters (to be used mostly when extending this method)
       @type parameters: list of tuples
       @param paramValues: Optional list of parameters values (to be used mostly when extending this method)
       @type parameters: list of tuples
    """
        kwargs = {"executable": executable, "arguments": arguments, "logFile": logFile}
        if not type(executable) == type(" ") or not type(arguments) == type(" ") or not type(logFile) == type(" "):
            return self._reportError("Expected strings for executable and arguments", **kwargs)

        if os.path.exists(executable):
            self.log.verbose("Found script executable file %s" % (executable))
            self.addToInputSandbox.append(executable)
            logName = "%s.log" % (os.path.basename(executable))
        else:
            self.log.warn("The executable code could not be found locally")
            logName = "CodeOutput.log"

        self.stepCount += 1
        stepName = "RunScriptStep%s" % (self.stepCount)

        if logFile:
            if type(logFile) == type(" "):
                logName = str(logFile)
        else:
            logName = "Script%s_%s" % (self.stepCount, logName)

        if not modulesList:
            modulesList = ["Script"]
        if not parameters:
            parameters = [
                ("executable", "string", "", "Executable Script"),
                ("arguments", "string", "", "Arguments for executable Script"),
                ("applicationLog", "string", "", "Log file name"),
            ]

        step = getStepDefinition("ScriptStep%s" % (self.stepCount), modulesList, parametersList=parameters)
        self.addToOutputSandbox.append(logName)

        stepInstance = addStepToWorkflow(self.workflow, step, stepName)

        stepInstance.setValue("applicationLog", logName)
        stepInstance.setValue("executable", executable)
        if arguments:
            stepInstance.setValue("arguments", arguments)
        if paramValues:
            for param, value in paramValues:
                stepInstance.setValue(param, value)

        return S_OK(stepInstance)
コード例 #14
0
  def setExecutable(self, executable, arguments='', logFile='',
                    modulesList=None, parameters=None, paramValues=None):
    """Helper function.

       Specify executable script to run with optional arguments and log file
       for standard output.

       These can be either:

       - Submission of a python or shell script to DIRAC
          - Can be inline scripts e.g. C{'/bin/ls'}
          - Scripts as executables e.g. python or shell script file

       Example usage:

       >>> job = Job()
       >>> job.setExecutable('myScript.py')

       :param executable: Executable
       :type executable: string
       :param arguments: Optional arguments to executable
       :type arguments: string
       :param logFile: Optional log file name
       :type logFile: string
       :param modulesList: Optional list of modules (to be used mostly when extending this method)
       :type modulesList: python:list
       :param parameters: Optional list of parameters (to be used mostly when extending this method)
       :type parameters: python:list of tuples
       :param paramValues: Optional list of parameters values (to be used mostly when extending this method)
       :type parameters: python:list of tuples
    """
    kwargs = {'executable': executable, 'arguments': arguments, 'logFile': logFile}
    if not isinstance(executable, basestring) or not isinstance(arguments, six.string_types) or \
       not isinstance(logFile, six.string_types):
      return self._reportError('Expected strings for executable and arguments', **kwargs)

    if os.path.exists(executable):
      self.log.verbose('Found script executable file %s' % (executable))
      self.addToInputSandbox.append(executable)
      logName = '%s.log' % (os.path.basename(executable))
    else:
      self.log.warn('The executable code could not be found locally')
      logName = 'CodeOutput.log'

    self.stepCount += 1
    stepName = 'RunScriptStep%s' % (self.stepCount)

    if logFile:
      if isinstance(logFile, six.string_types):
        logName = str(logFile)
    else:
      logName = "Script%s_%s" % (self.stepCount, logName)

    if not modulesList:
      modulesList = ['Script']
    if not parameters:
      parameters = [('executable', 'string', '', "Executable Script"),
                    ('arguments', 'string', '', 'Arguments for executable Script'),
                    ('applicationLog', 'string', '', "Log file name")]

    step = getStepDefinition('ScriptStep%s' % (self.stepCount), modulesList, parametersList=parameters)
    self.addToOutputSandbox.append(logName)

    stepInstance = addStepToWorkflow(self.workflow, step, stepName)

    stepInstance.setValue('applicationLog', logName)
    stepInstance.setValue('executable', executable)
    if arguments:
      # If arguments are expressed in terms of parameters, pass them to Workflow
      # These arguments will be resolved on the server side for each parametric job
      if re.search(r'%\(.*\)s', arguments) or re.search('%n', arguments):
        self.parametricWFArguments['arguments'] = arguments
      else:
        stepInstance.setValue('arguments', arguments)
    if paramValues:
      for param, value in paramValues:
        stepInstance.setValue(param, value)

    return S_OK(stepInstance)