def addWrapper(self, logFile = ''): """ Overload the DIRAC.Job.setExecutable """ logFile = str(logFile) stepDefn = 'WrapperStep' stepName = 'RunWrapperStep' moduleName = 'GlastWrapperCall' module = ModuleDefinition( moduleName ) module.setDescription( 'The utility that calls the pipeline_wrapper.' ) body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n' module.setBody( body ) # Create Step definition step = StepDefinition( stepDefn ) step.addModule( module ) #moduleInstance = step.createModuleInstance( 'GlastWrapperCall', stepDefn ) # Define step parameters step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) self.addToOutputSandbox.append( logFile ) self.workflow.addStep( step ) # Define Step and its variables stepInstance = self.workflow.createStepInstance( stepDefn, stepName ) stepInstance.setValue( "logFile", logFile ) return S_OK()
def _addToWorkflow(self): """ This is called just before submission. It creates the actual workflow. The linking of parameters can only be done here """ for application in self.applicationlist: #Start by defining step number self.stepCount += 1 res = application._analyseJob(self) if not res['OK']: return res res = application._checkWorkflowConsistency() if not res['OK']: self.log.error("%s failed to check its consistency:" % application, "%s" % res['Message']) return S_ERROR("%s failed to check its consistency: %s" % (application, res['Message'])) ##Now we can create the step and add it to the workflow #First we need a unique name, let's use the application name and step number stepname = "%s_step_%s" % (application.appname, self.stepCount) stepdefinition = StepDefinition(stepname) self.steps.append(stepdefinition) ##Set the modules needed by the application res = self._jobSpecificModules(application, stepdefinition) if not res['OK']: self.log.error("Failed to add modules:", "%s" % res['Message']) return S_ERROR("Failed to add modules: %s" % res['Message']) ### add the parameters to the step res = application._addParametersToStep(stepdefinition) if not res['OK']: self.log.error("Failed to add parameters:", "%s" % res['Message']) return S_ERROR("Failed to add parameters: %s" % res['Message']) ##Now the step is defined, let's add it to the workflow self.workflow.addStep(stepdefinition) ###Now we need to get a step instance object to set the parameters' values stepInstance = self.workflow.createStepInstance(stepdefinition.getType(), stepname) ##Set the parameters values to the step instance res = application._setStepParametersValues(stepInstance) if not res['OK']: self.log.error("Failed to resolve parameters values:", "%s" % res['Message']) return S_ERROR("Failed to resolve parameters values: %s" % res['Message']) res = application._resolveLinkedStepParameters(stepInstance) if not res['OK']: self.log.error("Failed to resolve linked parameters:", "%s" % res['Message']) return S_ERROR("Failed to resolve linked parameters: %s" % res['Message']) #Now prevent overwriting of parameter values. application._addedtojob() self._addParameter(self.workflow, 'TotalSteps', 'String', self.stepCount, 'Total number of steps') return S_OK()
def getStepDefinition(stepName, modulesNameList=None, importLine="""""", parametersList=None): """ Given a name, a list of modules name, and a list of parameters, returns a step definition. Remember that Step definition = Parameters + Module Instances """ if modulesNameList is None: modulesNameList = [] if parametersList is None: parametersList = [] # In case the importLine is not set, this is looking for a DIRAC extension, if any. # The extension is supposed to be called ExtDIRAC. if not importLine: importLine = "DIRAC.Workflow.Modules" for ext in getCSExtensions(): if ext.lower() == getVO(): importLine = ext + "DIRAC.Workflow.Modules" break stepDef = StepDefinition(stepName) for moduleName in modulesNameList: # create the module definition moduleDef = ModuleDefinition(moduleName) try: # Look in the importLine given, or the DIRAC if the given location can't be imported moduleDef.setDescription( getattr( __import__("%s.%s" % (importLine, moduleName), globals(), locals(), ['__doc__']), "__doc__")) moduleDef.setBody("""\nfrom %s.%s import %s\n""" % (importLine, moduleName, moduleName)) except ImportError: alternativeImportLine = "DIRAC.Workflow.Modules" moduleDef.setDescription( getattr( __import__("%s.%s" % (alternativeImportLine, moduleName), globals(), locals(), ['__doc__']), "__doc__")) moduleDef.setBody("""\nfrom %s.%s import %s\n""" % (alternativeImportLine, moduleName, moduleName)) # add the module to the step, and instance it stepDef.addModule(moduleDef) stepDef.createModuleInstance(module_type=moduleName, name=moduleName) # add parameters to the module definition for pName, pType, pValue, pDesc in parametersList: p = Parameter(pName, pValue, pType, "", "", True, False, pDesc) stepDef.addParameter(Parameter(parameter=p)) return stepDef
def addWrapper(self, logFile=''): """ Overload the DIRAC.Job.setExecutable """ logFile = str(logFile) stepDefn = 'WrapperStep' stepName = 'RunWrapperStep' moduleName = 'GlastWrapperCall' module = ModuleDefinition(moduleName) module.setDescription('The utility that calls the pipeline_wrapper.') body = 'from GlastDIRAC.PipelineSystem.Modules.GlastWrapperCall import GlastWrapperCall\n' module.setBody(body) # Create Step definition step = StepDefinition(stepDefn) step.addModule(module) moduleInstance = step.createModuleInstance('GlastWrapperCall', stepDefn) # Define step parameters step.addParameter( Parameter("logFile", "", "string", "", "", False, False, 'Log file name')) self.addToOutputSandbox.append(logFile) self.workflow.addStep(step) # Define Step and its variables stepInstance = self.workflow.createStepInstance(stepDefn, stepName) stepInstance.setValue("logFile", logFile) return S_OK()
def createSingleModuleWorkflow(module, name): ''' Creates a workflow based on a single module definition ''' moduleType = module.getType() moduleName = name workflow = Workflow() step = StepDefinition(moduleType + '_step') step.addModule(module) moduleInstance = step.createModuleInstance(moduleType, moduleName) step.addParameter(moduleInstance.parameters.getInput()) workflow.addParameter(moduleInstance.parameters.getInput()) workflow.addStep(step) stepInstance = workflow.createStepInstance(moduleType + '_step', moduleName + '_step') # Propagate the module input parameters to the workflow level moduleInstance.linkParameterUp(moduleInstance.parameters.getInput()) stepInstance.linkParameterUp(moduleInstance.parameters.getInput()) workflow.setName(name) workflow.setDescription('Single module workflow from ' + moduleType + ' type module') workflow.setDescrShort(moduleType + ' workflow') return workflow
def createSingleModuleWorkflow(module,name): ''' Creates a workflow based on a single module definition ''' moduleType = module.getType() moduleName = name workflow = Workflow() step = StepDefinition(moduleType+'_step') step.addModule(module) moduleInstance = step.createModuleInstance(moduleType,moduleName) step.addParameter(moduleInstance.parameters.getInput()) workflow.addParameter(moduleInstance.parameters.getInput()) workflow.addStep(step) stepInstance = workflow.createStepInstance(moduleType+'_step',moduleName+'_step') # Propagate the module input parameters to the workflow level moduleInstance.linkParameterUp(moduleInstance.parameters.getInput()) stepInstance.linkParameterUp(moduleInstance.parameters.getInput()) workflow.setName(name) workflow.setDescription('Single module workflow from '+moduleType+' type module') workflow.setDescrShort(moduleType+' workflow') return workflow
def getStepDefinition(stepName, modulesNameList=None, importLine="", parametersList=None): """Given a name, a list of modules name, and a list of parameters, returns a step definition. Remember that Step definition = Parameters + Module Instances """ if modulesNameList is None: modulesNameList = [] if parametersList is None: parametersList = [] stepDef = StepDefinition(stepName) for moduleName in modulesNameList: module = None if importLine: try: module = importlib.import_module(importLine + "." + moduleName) except ImportError: pass # In case the importLine is not set, this is looking for a DIRAC extension, if any if module is None: module = ObjectLoader().loadModule("Workflow.Modules." + moduleName)["Value"] # create the module definition moduleDef = ModuleDefinition(moduleName) moduleDef.setDescription(module.__doc__) moduleDef.setBody("\nfrom %s import %s\n" % (module.__name__, moduleName)) # add the module to the step, and instance it stepDef.addModule(moduleDef) stepDef.createModuleInstance(module_type=moduleName, name=moduleName) # add parameters to the module definition for pName, pType, pValue, pDesc in parametersList: p = Parameter(pName, pValue, pType, "", "", True, False, pDesc) stepDef.addParameter(Parameter(parameter=p)) return stepDef
def startElement(self, name, attrs): # print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames() self.clearCharacters( ) # clear to remove empty or nonprintable characters if name == "Workflow": if self.root == None: # if root not defined by constractor self.root = Workflow() self.stack.append(self.root) elif name == "StepDefinition": obj = StepDefinition("TemporaryXMLObject_StepDefinition") if self.root == None: # in case we are saving Step only self.root = obj self.stack.append(obj) elif name == "StepInstance": obj = StepInstance("TemporaryXMLObject_StepInstance") self.stack.append(obj) elif name == "ModuleDefinition": obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition") if self.root == None: # in case we are saving Module only self.root = obj self.stack.append(obj) elif name == "ModuleInstance": obj = ModuleInstance("TemporaryXMLObject_ModuleInstance") self.stack.append(obj) elif name == "Parameter": obj = Parameter(str(attrs['name']), None, str(attrs['type']), str(attrs['linked_module']), str(attrs['linked_parameter']), str(attrs['in']), str(attrs['out']), str(attrs['description'])) self.stack.append(obj) # TEMPORARY CODE elif name == "origin" or name == "version" or name == "name" or name == "type" or name == "value" or\ name == "required" or name == "descr_short" or name == "name" or name == "type" or name == "description" or name == "body": pass else: print "UNTREATED! startElement name=", name, "attr=", attrs.getLength( ), attrs.getNames() pass
def getStepDefinition( stepName, modulesNameList = [], importLine = """""", parametersList = [] ): """ Given a name, a list of modules name, and a list of parameters, returns a step definition. Remember that Step definition = Parameters + Module Instances """ # In case the importLine is not set, this is looking for a DIRAC extension, if any. # The extension is supposed to be called ExtDIRAC. if not importLine: importLine = "DIRAC.Workflow.Modules" for ext in getCSExtensions(): if ext.lower() == getVO(): importLine = ext + "DIRAC.Workflow.Modules" break stepDef = StepDefinition( stepName ) for moduleName in modulesNameList: # create the module definition moduleDef = ModuleDefinition( moduleName ) try: # Look in the importLine given, or the DIRAC if the given location can't be imported moduleDef.setDescription( getattr( __import__( "%s.%s" % ( importLine, moduleName ), globals(), locals(), ['__doc__'] ), "__doc__" ) ) moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( importLine, moduleName, moduleName ) ) except ImportError: alternativeImportLine = "DIRAC.Workflow.Modules" moduleDef.setDescription( getattr( __import__( "%s.%s" % ( alternativeImportLine, moduleName ), globals(), locals(), ['__doc__'] ), "__doc__" ) ) moduleDef.setBody( """\nfrom %s.%s import %s\n""" % ( alternativeImportLine, moduleName, moduleName ) ) # add the module to the step, and instance it stepDef.addModule( moduleDef ) stepDef.createModuleInstance( module_type = moduleName, name = moduleName ) # add parameters to the module definition for pName, pType, pValue, pDesc in parametersList: p = Parameter( pName, pValue, pType, "", "", True, False, pDesc ) stepDef.addParameter( Parameter( parameter = p ) ) return stepDef
def _addRealFinalization(self): """ See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition """ importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>' dataUpload = ModuleDefinition('UploadOutputData') dataUpload.setDescription('Uploads the output data') self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag') body = string.replace(importLine, '<MODULE>', 'UploadOutputData') dataUpload.setBody(body) failoverRequest = ModuleDefinition('FailoverRequest') failoverRequest.setDescription('Sends any failover requests') self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag') body = string.replace(importLine, '<MODULE>', 'FailoverRequest') failoverRequest.setBody(body) registerdata = ModuleDefinition('ILDRegisterOutputData') registerdata.setDescription( 'Module to add in the metadata catalog the relevant info about the files' ) self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag') body = string.replace(importLine, '<MODULE>', 'ILDRegisterOutputData') registerdata.setBody(body) logUpload = ModuleDefinition('UploadLogFile') logUpload.setDescription('Uploads the output log files') self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag') body = string.replace(importLine, '<MODULE>', 'UploadLogFile') logUpload.setBody(body) finalization = StepDefinition('Job_Finalization') finalization.addModule(dataUpload) up = finalization.createModuleInstance('UploadOutputData', 'dataUpload') up.setValue("enable", self.finalsdict['uploadData']) finalization.addModule(registerdata) # TODO: create ILDRegisterOutputData ro = finalization.createModuleInstance('ILDRegisterOutputData', 'ILDRegisterOutputData') ro.setValue("enable", self.finalsdict['registerData']) finalization.addModule(logUpload) ul = finalization.createModuleInstance('UploadLogFile', 'logUpload') ul.setValue("enable", self.finalsdict['uploadLog']) finalization.addModule(failoverRequest) fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest') fr.setValue("enable", self.finalsdict['sendFailover']) self.workflow.addStep(finalization) self.workflow.createStepInstance('Job_Finalization', 'finalization') return S_OK()
def test__getStepDefinition( self ): importLine = """ from DIRAC.Workflow.Modules.<MODULE> import <MODULE> """ # modules gaudiApp = ModuleDefinition( 'Script' ) body = importLine.replace( '<MODULE>', 'Script' ) gaudiApp.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'Script' ), globals(), locals(), ['__doc__'] ), "__doc__" ) ) gaudiApp.setBody( body ) genBKReport = ModuleDefinition( 'FailoverRequest' ) body = importLine.replace( '<MODULE>', 'FailoverRequest' ) genBKReport.setDescription( getattr( __import__( "%s.%s" % ( 'DIRAC.Workflow.Modules', 'FailoverRequest' ), globals(), locals(), ['__doc__'] ), "__doc__" ) ) genBKReport.setBody( body ) # step appDefn = StepDefinition( 'App_Step' ) appDefn.addModule( gaudiApp ) appDefn.createModuleInstance( 'Script', 'Script' ) appDefn.addModule( genBKReport ) appDefn.createModuleInstance( 'FailoverRequest', 'FailoverRequest' ) appDefn.addParameterLinked( gaudiApp.parameters ) stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'] ) self.assertTrue( str( appDefn ) == str( stepDef ) ) self.job._addParameter( appDefn, 'name', 'type', 'value', 'desc' ) self.job._addParameter( appDefn, 'name1', 'type1', 'value1', 'desc1' ) stepDef = getStepDefinition( 'App_Step', ['Script', 'FailoverRequest'], parametersList = [[ 'name', 'type', 'value', 'desc' ], [ 'name1', 'type1', 'value1', 'desc1' ]] ) self.assertTrue( str( appDefn ) == str( stepDef ) )
def _addRealFinalization(self): """ This is called at creation: now that the workflow is created at the last minute, we need to add this also at the last minute """ importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>' dataUpload = ModuleDefinition('UploadOutputData') dataUpload.setDescription('Uploads the output data') self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'UploadOutputData') dataUpload.setBody(body) failoverRequest = ModuleDefinition('FailoverRequest') failoverRequest.setDescription('Sends any failover requests') self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'FailoverRequest') failoverRequest.setBody(body) registerdata = ModuleDefinition('RegisterOutputData') registerdata.setDescription( 'Module to add in the metadata catalog the relevant info about the files' ) self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'RegisterOutputData') registerdata.setBody(body) logUpload = ModuleDefinition('UploadLogFile') logUpload.setDescription('Uploads the output log files') self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'UploadLogFile') logUpload.setBody(body) finalization = StepDefinition('Job_Finalization') finalization.addModule(dataUpload) up = finalization.createModuleInstance('UploadOutputData', 'dataUpload') up.setValue("enable", self.finalsdict['uploadData']) finalization.addModule(registerdata) ro = finalization.createModuleInstance('RegisterOutputData', 'RegisterOutputData') ro.setValue("enable", self.finalsdict['registerData']) finalization.addModule(logUpload) ul = finalization.createModuleInstance('UploadLogFile', 'logUpload') ul.setValue("enable", self.finalsdict['uploadLog']) finalization.addModule(failoverRequest) fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest') fr.setValue("enable", self.finalsdict['sendFailover']) self.workflow.addStep(finalization) self.workflow.createStepInstance('Job_Finalization', 'finalization') return S_OK()
def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step
def test__getStepDefinition(self): importLine = """ from DIRAC.Workflow.Modules.<MODULE> import <MODULE> """ # modules gaudiApp = ModuleDefinition("Script") body = importLine.replace("<MODULE>", "Script") gaudiApp.setDescription( getattr( __import__("%s.%s" % ("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), "__doc__" ) ) gaudiApp.setBody(body) genBKReport = ModuleDefinition("FailoverRequest") body = importLine.replace("<MODULE>", "FailoverRequest") genBKReport.setDescription( getattr( __import__("%s.%s" % ("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"]), "__doc__", ) ) genBKReport.setBody(body) # step appDefn = StepDefinition("App_Step") appDefn.addModule(gaudiApp) appDefn.createModuleInstance("Script", "Script") appDefn.addModule(genBKReport) appDefn.createModuleInstance("FailoverRequest", "FailoverRequest") appDefn.addParameterLinked(gaudiApp.parameters) stepDef = getStepDefinition("App_Step", ["Script", "FailoverRequest"]) self.assertTrue(str(appDefn) == str(stepDef)) self.job._addParameter(appDefn, "name", "type", "value", "desc") self.job._addParameter(appDefn, "name1", "type1", "value1", "desc1") stepDef = getStepDefinition( "App_Step", ["Script", "FailoverRequest"], parametersList=[["name", "type", "value", "desc"], ["name1", "type1", "value1", "desc1"]], ) self.assertTrue(str(appDefn) == str(stepDef))
def append(self,application): """ Helper function This is the main part: call for every application @param application: Application instance """ #Start by defining step number self.stepnumber = len(self.steps) + 1 res = application._analyseJob(self) if not res['OK']: return res res = application._checkConsistency() if not res['OK']: self.log.error("%s failed to check its consistency: %s"%(application,res['Message'])) return S_ERROR("%s failed to check its consistency: %s"%(application,res['Message'])) res = self._jobSpecificParams(application) if not res['OK']: self.log.error("%s failed job specific checks: %s"%(application,res['Message'])) return S_ERROR("%s failed job specific checks: %s"%(application,res['Message'])) res = application._checkFinalConsistency() if not res['OK']: self.log.error("%s failed to check its consistency: %s"%(application,res['Message'])) return S_ERROR("%s failed to check its consistency: %s"%(application,res['Message'])) ### Once the consistency has been checked, we can add the application to the list of apps. self.applicationlist.append(application) ##Get the application's sandbox and add it to the job's for isb in application.inputSB: if not isb in self.inputsandbox: self.inputsandbox.append(isb) #self.inputsandbox.extend(application.inputSB) ##Now we can create the step and add it to the workflow #First we need a unique name, let's use the application name and step number stepname = "%s_step_%s"%(application.appname,self.stepnumber) stepdefinition = StepDefinition(stepname) self.steps.append(stepdefinition) ##Set the modules needed by the application res = self._jobSpecificModules(application,stepdefinition) if not res['OK']: self.log.error("Failed to add modules: %s"%res['Message']) return S_ERROR("Failed to add modules: %s"%res['Message']) ### add the parameters to the step res = application._addParametersToStep(stepdefinition) if not res['OK']: self.log.error("Failed to add parameters: %s"%res['Message']) return S_ERROR("Failed to add parameters: %s"%res['Message']) ##Now the step is defined, let's add it to the workflow self.workflow.addStep(stepdefinition) ###Now we need to get a step instance object to set the parameters' values stepInstance = self.workflow.createStepInstance(stepdefinition.getType(),stepname) ##Set the parameters values to the step instance res = application._setStepParametersValues(stepInstance) if not res['OK']: self.log.error("Failed to resolve parameters values: %s"%res['Message']) return S_ERROR("Failed to resolve parameters values: %s"%res['Message']) ##stepInstance.setLink("InputFile",here lies the step name of the linked step, maybe get it from the application,"OutputFile") res = application._resolveLinkedStepParameters(stepInstance) if not res['OK']: self.log.error("Failed to resolve linked parameters: %s"%res['Message']) return S_ERROR("Failed to resolve linked parameters: %s"%res['Message']) #Now prevent overwriting of parameter values. application._addedtojob() self._addParameter(self.workflow, 'TotalSteps', 'String', self.stepnumber, 'Total number of steps') if application.nbevts: self._addParameter(self.workflow, 'NbOfEvts', 'int', application.nbevts, "Number of events to process") ##Finally, add the software packages if needed if application.appname and application.version: self._addSoftware(application.appname, application.version) return S_OK()
def test__getStepDefinition(self): importLine = """ from DIRAC.Workflow.Modules.<MODULE> import <MODULE> """ # modules gaudiApp = ModuleDefinition('Script') body = importLine.replace('<MODULE>', 'Script') gaudiApp.setDescription(getattr(__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'Script'), globals(), locals(), ['__doc__']), "__doc__")) gaudiApp.setBody(body) genBKReport = ModuleDefinition('FailoverRequest') body = importLine.replace('<MODULE>', 'FailoverRequest') genBKReport.setDescription(getattr(__import__("%s.%s" % ('DIRAC.Workflow.Modules', 'FailoverRequest'), globals(), locals(), ['__doc__']), "__doc__")) genBKReport.setBody(body) # step appDefn = StepDefinition('App_Step') appDefn.addModule(gaudiApp) appDefn.createModuleInstance('Script', 'Script') appDefn.addModule(genBKReport) appDefn.createModuleInstance('FailoverRequest', 'FailoverRequest') appDefn.addParameterLinked(gaudiApp.parameters) stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest']) self.assertTrue(str(appDefn) == str(stepDef)) self.job._addParameter(appDefn, 'name', 'type', 'value', 'desc') self.job._addParameter(appDefn, 'name1', 'type1', 'value1', 'desc1') stepDef = getStepDefinition('App_Step', ['Script', 'FailoverRequest'], parametersList=[['name', 'type', 'value', 'desc'], ['name1', 'type1', 'value1', 'desc1']]) self.assertTrue(str(appDefn) == str(stepDef))
def _addToWorkflow(self): """ This is called just before submission. It creates the actual workflow. The linking of parameters can only be done here """ for application in self.applicationlist: #Start by defining step number self.stepCount += 1 res = application._analyseJob(self) if not res['OK']: return res res = application._checkWorkflowConsistency() if not res['OK']: self.log.error( "%s failed to check its consistency:" % application, "%s" % res['Message']) return S_ERROR("%s failed to check its consistency: %s" % (application, res['Message'])) ##Now we can create the step and add it to the workflow #First we need a unique name, let's use the application name and step number stepname = "%s_step_%s" % (application.appname, self.stepCount) stepdefinition = StepDefinition(stepname) self.steps.append(stepdefinition) ##Set the modules needed by the application res = self._jobSpecificModules(application, stepdefinition) if not res['OK']: self.log.error("Failed to add modules:", "%s" % res['Message']) return S_ERROR("Failed to add modules: %s" % res['Message']) ### add the parameters to the step res = application._addParametersToStep(stepdefinition) if not res['OK']: self.log.error("Failed to add parameters:", "%s" % res['Message']) return S_ERROR("Failed to add parameters: %s" % res['Message']) ##Now the step is defined, let's add it to the workflow self.workflow.addStep(stepdefinition) ###Now we need to get a step instance object to set the parameters' values stepInstance = self.workflow.createStepInstance( stepdefinition.getType(), stepname) ##Set the parameters values to the step instance res = application._setStepParametersValues(stepInstance) if not res['OK']: self.log.error("Failed to resolve parameters values:", "%s" % res['Message']) return S_ERROR("Failed to resolve parameters values: %s" % res['Message']) res = application._resolveLinkedStepParameters(stepInstance) if not res['OK']: self.log.error("Failed to resolve linked parameters:", "%s" % res['Message']) return S_ERROR("Failed to resolve linked parameters: %s" % res['Message']) #Now prevent overwriting of parameter values. application._addedtojob() self._addParameter(self.workflow, 'TotalSteps', 'String', self.stepCount, 'Total number of steps') return S_OK()
def _addRealFinalization( self ): """ See :mod:`~ILCDIRAC.Interfaces.API.NewInterface.ProductionJob` for definition """ importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>' dataUpload = ModuleDefinition( 'UploadOutputData' ) dataUpload.setDescription( 'Uploads the output data' ) self._addParameter( dataUpload, 'enable', 'bool', False, 'EnableFlag' ) body = string.replace( importLine, '<MODULE>', 'UploadOutputData' ) dataUpload.setBody( body ) failoverRequest = ModuleDefinition( 'FailoverRequest' ) failoverRequest.setDescription( 'Sends any failover requests' ) self._addParameter( failoverRequest, 'enable', 'bool', False, 'EnableFlag' ) body = string.replace( importLine, '<MODULE>', 'FailoverRequest' ) failoverRequest.setBody( body ) registerdata = ModuleDefinition( 'ILDRegisterOutputData' ) registerdata.setDescription( 'Module to add in the metadata catalog the relevant info about the files' ) self._addParameter( registerdata, 'enable', 'bool', False, 'EnableFlag' ) body = string.replace( importLine, '<MODULE>', 'ILDRegisterOutputData' ) registerdata.setBody( body ) logUpload = ModuleDefinition( 'UploadLogFile' ) logUpload.setDescription( 'Uploads the output log files' ) self._addParameter( logUpload, 'enable', 'bool', False, 'EnableFlag' ) body = string.replace( importLine, '<MODULE>', 'UploadLogFile' ) logUpload.setBody( body ) finalization = StepDefinition( 'Job_Finalization' ) finalization.addModule( dataUpload ) up = finalization.createModuleInstance( 'UploadOutputData', 'dataUpload' ) up.setValue( "enable", self.finalsdict['uploadData'] ) finalization.addModule( registerdata ) # TODO: create ILDRegisterOutputData ro = finalization.createModuleInstance( 'ILDRegisterOutputData', 'ILDRegisterOutputData' ) ro.setValue( "enable", self.finalsdict['registerData'] ) finalization.addModule( logUpload ) ul = finalization.createModuleInstance( 'UploadLogFile', 'logUpload' ) ul.setValue( "enable", self.finalsdict['uploadLog'] ) finalization.addModule( failoverRequest ) fr = finalization.createModuleInstance( 'FailoverRequest', 'failoverRequest' ) fr.setValue( "enable", self.finalsdict['sendFailover'] ) self.workflow.addStep( finalization ) self.workflow.createStepInstance( 'Job_Finalization', 'finalization' ) return S_OK()
def addFinalization(self, uploadData=False, registerData=False, uploadLog = False, sendFailover=False): """ Add finalization step @param uploadData: Upload or not the data to the storage @param uploadLog: Upload log file to storage (currently only available for admins, thus add them to OutputSandbox) @param sendFailover: Send Failover requests, and declare files as processed or unused in transfDB @param registerData: Register data in the file catalog @todo: Do the registration only once, instead of once for each job """ self.importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>' dataUpload = ModuleDefinition('UploadOutputData') dataUpload.setDescription('Uploads the output data') self._addParameter(dataUpload,'enable','bool',False,'EnableFlag') body = string.replace(self.importLine,'<MODULE>','UploadOutputData') dataUpload.setBody(body) failoverRequest = ModuleDefinition('FailoverRequest') failoverRequest.setDescription('Sends any failover requests') self._addParameter(failoverRequest,'enable','bool',False,'EnableFlag') body = string.replace(self.importLine,'<MODULE>','FailoverRequest') failoverRequest.setBody(body) registerdata = ModuleDefinition('DBDGenRegisterOutputData') registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files') self._addParameter(registerdata,'enable','bool',False,'EnableFlag') body = string.replace(self.importLine,'<MODULE>','DBDGenRegisterOutputData') registerdata.setBody(body) logUpload = ModuleDefinition('UploadLogFile') logUpload.setDescription('Uploads the output log files') self._addParameter(logUpload,'enable','bool',False,'EnableFlag') body = string.replace(self.importLine,'<MODULE>','UploadLogFile') logUpload.setBody(body) finalization = StepDefinition('Job_Finalization') finalization.addModule(dataUpload) up = finalization.createModuleInstance('UploadOutputData','dataUpload') up.setValue("enable",uploadData) finalization.addModule(registerdata) ro = finalization.createModuleInstance('DBDGenRegisterOutputData','DBDGenRegisterOutputData') ro.setValue("enable",registerData) finalization.addModule(logUpload) ul = finalization.createModuleInstance('UploadLogFile','logUpload') ul.setValue("enable",uploadLog) finalization.addModule(failoverRequest) fr = finalization.createModuleInstance('FailoverRequest','failoverRequest') fr.setValue("enable",sendFailover) self.workflow.addStep(finalization) self.workflow.createStepInstance('Job_Finalization', 'finalization') return S_OK()
def _addRealFinalization(self): """ This is called at creation: now that the workflow is created at the last minute, we need to add this also at the last minute """ importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>' dataUpload = ModuleDefinition('UploadOutputData') dataUpload.setDescription('Uploads the output data') self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'UploadOutputData') dataUpload.setBody(body) failoverRequest = ModuleDefinition('FailoverRequest') failoverRequest.setDescription('Sends any failover requests') self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'FailoverRequest') failoverRequest.setBody(body) registerdata = ModuleDefinition('RegisterOutputData') registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files') self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'RegisterOutputData') registerdata.setBody(body) logUpload = ModuleDefinition('UploadLogFile') logUpload.setDescription('Uploads the output log files') self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'UploadLogFile') logUpload.setBody(body) errorReport = ModuleDefinition('ReportErrors') errorReport.setDescription('Reports errors at the end') body = importLine.replace('<MODULE>', 'ReportErrors') errorReport.setBody(body) finalization = StepDefinition('Job_Finalization') finalization.addModule(dataUpload) up = finalization.createModuleInstance('UploadOutputData', 'dataUpload') up.setValue("enable", self.finalsdict['uploadData']) finalization.addModule(registerdata) ro = finalization.createModuleInstance('RegisterOutputData', 'RegisterOutputData') ro.setValue("enable", self.finalsdict['registerData']) finalization.addModule(logUpload) ul = finalization.createModuleInstance('UploadLogFile', 'logUpload') ul.setValue("enable", self.finalsdict['uploadLog']) finalization.addModule(failoverRequest) fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest') fr.setValue("enable", self.finalsdict['sendFailover']) finalization.addModule(errorReport) fr = finalization.createModuleInstance('ReportErrors', 'reportErrors') self.workflow.addStep(finalization) self.workflow.createStepInstance('Job_Finalization', 'finalization') return S_OK()