def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ): """Instantiates the Workflow object and some default parameters. """ self.log = gLogger self.section = COMPONENT_NAME self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True #gConfig.getValue('Tier0SE-tape','SEName') self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo( disableVOMS = True ) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup( ret['Value']['group'] ) self.group = vo self.site = 'ANY' #ANY #self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] self.systemConfig = 'ANY' self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE', 'MinCPUTime': 'other.NAME<=VALUE', 'Site': 'other.NAME=="VALUE"', 'Platform': 'other.NAME=="VALUE"', #'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately 'BannedSites': 'other.Site!="VALUE"', 'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'} ##Add member to handle Parametric jobs self.parametric = {} self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow( script ) #Global error dictionary self.errorDict = {}
def startElement(self, name, attrs): #print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames() self.clearCharacters() # clear to remove empty or nonprintable characters if name == "Workflow": if self.root == None: #if root not defined by constractor self.root = Workflow() self.stack.append(self.root) elif name == "StepDefinition": obj = StepDefinition("TemporaryXMLObject_StepDefinition") if self.root == None: # in case we are saving Step only self.root = obj self.stack.append(obj) elif name == "StepInstance": obj = StepInstance("TemporaryXMLObject_StepInstance") self.stack.append(obj) elif name == "ModuleDefinition": obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition") if self.root == None: # in case we are saving Module only self.root = obj self.stack.append(obj) elif name == "ModuleInstance": obj = ModuleInstance("TemporaryXMLObject_ModuleInstance") self.stack.append(obj) elif name == "Parameter": obj = Parameter(str(attrs['name']), None, str(attrs['type']), str(attrs['linked_module']), str(attrs['linked_parameter']), str(attrs['in']), str(attrs['out']), str(attrs['description'])) self.stack.append(obj) # TEMPORARY CODE elif name=="origin" or name == "version" or name == "name" or name == "type" or name == "value" or\ name == "required" or name == "descr_short" or name == "name" or name == "type" or name == "description" or name == "body": pass else: print "UNTREATED! startElement name=", name, "attr=", attrs.getLength(), attrs.getNames() pass
def execute_module(self, script_to_source, fcc_executable, fcc_conf_file, fcc_input_files, fcc_output_file, fccsw, number_of_events): #define FCC Module fcc_module = ModuleDefinition( 'ExecutionModule' ) #during constraction class creates duplicating copies of the params fcc_module.setDescription('FCC module') fcc_module.setBody( 'from ILCDIRAC.Workflow.Modules.fcc_execution_module import ExecutionModule\n' ) fcc_module.addParameter( Parameter("script_to_source", script_to_source, "string", "", "", True, False, "The script to source")) fcc_module.addParameter( Parameter( "fcc_executable", fcc_executable, "string", "", "", True, False, "The executable you want to use ie. FCCSW,FCCPHYSICS...")) fcc_module.addParameter( Parameter("fcc_conf_file", fcc_conf_file, "string", "", "", True, False, "The configuration file you want to use.")) fcc_module.addParameter( Parameter( "fcc_input_files", fcc_input_files, "string", "", "", True, False, "The input files the job needs ie. --ifiles file1 file2 file3 ..." )) fcc_module.addParameter( Parameter("fcc_output_file", fcc_output_file, "string", "", "", True, False, "The output file you want to create.")) fcc_module.addParameter( Parameter("fccsw", fccsw, "string", "", "", True, False, "The use of fccsw software.")) fcc_module.addParameter( Parameter("number_of_events", number_of_events, "string", "", "", True, False, "Number of events you want to generate")) fcc_step = StepDefinition('FCC_App_Step') fcc_step.addModule( fcc_module) # Creating instance of the module 'FCC_App_Step' fcc_module_instance = fcc_step.createModuleInstance( 'ExecutionModule', 'fcc_module') #fcc_step.addParameterLinked(fcc_module.parameters) fcc_workflow = Workflow(name='FCC Workflow') fcc_workflow.setDescription('Workflow of FCC Applications') fcc_workflow.addStep(fcc_step) fcc_step_instance = fcc_workflow.createStepInstance( 'FCC_App_Step', 'fcc') #fcc_step_instance.addParameter(Parameter("fcc_executable","","string","","",True,False,"first event number")) #fcc_step_instance.setValue("fcc_executable", fcc_executable) print fcc_workflow.createCode() fcc_workflow.execute()
print( 'Test application execution' ) print 'Module MODULE.getDescrShort() =', self.MODULE.getDescrShort() print 'Module MODULE_DEFINITION_NAME =', self.MODULE_DEFINITION_NAME print 'Module MODULE_INSTANCE_NAME =', self.MODULE_INSTANCE_NAME print 'Module STEP.getDescrShort() =', self.STEP.getDescrShort() print 'Module STEP_DEFINITION_NAME =', self.STEP_DEFINITION_NAME print 'Module STEP_INSTANCE_NAME =', self.STEP_INSTANCE_NAME return 1""" md1 = ModuleDefinition('TestAppModule') md1.addParameter( Parameter("enable", "True", "bool", "", "", True, False, "If False execution is disabled")) md1.setBody(bodyTestApp) md1.setDescrShort('Empty Module') md1.setDescription('Empty Module to do some testing') md1.setRequired('') md1.setOrigin('') md1.setVersion(0.1) sd1 = StepDefinition('TestAppStep') sd1.addModule(md1) mi1 = sd1.createModuleInstance('TestAppModule', 'testappmod1') wf1 = Workflow('TestAppWF') wf1.addStep(sd1) si1 = wf1.createStepInstance('TestAppStep', 'testappstep1') print(wf1.createCode()) #eval(compile(wf1.createCode(),'<string>','exec')) wf1.execute()
opt_brunel = opt_brunel + ";EventLoopMgr.OutputLevel = 3" opt_brunel = opt_brunel + ";DstWriter.Output = \"DATAFILE=\'PFN:@{outputData}\' TYP=\'POOL_ROOTTREE\' OPT=\'RECREATE\'\"" opt_brunel = opt_brunel + ";EvtTupleSvc.Output = {\"EVTTAGS2 DATAFILE=\'PFN:@{etcf}\' TYP=\'POOL_ROOTTREE\' OPT=\'RECREATE\'\"}" #opt_brunel = opt_brunel+";EventSelector.Input = {\"COLLECTION=\'TagCreator/1\' DATAFILE=\'@{InputData}\' TYPE=\'POOL_ROOTTREE\' SEL=\'(GlobalOr>=1)\' OPT=\'READ\'\"}" opt_brunel = opt_brunel + ";EventPersistencySvc.CnvServices += { \"LHCb::RawDataCnvSvc\" }" opt_brunel = opt_brunel + ";ApplicationMgr.TopAlg += {\"StoreExplorerAlg\"}" opt_brunel = opt_brunel + ";StoreExplorerAlg.Load = 1" opt_brunel = opt_brunel + ";StoreExplorerAlg.PrintFreq = 0.99" opt_brunel = opt_brunel + ";IODataManager.AgeLimit = 2" step3 = StepDefinition('Job_Finalization') step3.addModule(module6) moduleInstance6 = step3.createModuleInstance('JobFinalization', 'module6') ############## WORKFLOW ################################# workflow1 = Workflow(name='CCRC-joel-test') workflow1.setDescription('Workflow of GaudiApplication') workflow1.addStep(step1) step1_prefix = "step1_" stepInstance1 = workflow1.createStepInstance('Gaudi_App_Step', 'Step1') # lets link all parameters them up with the level of workflow stepInstance1.setLink("systemConfig", "self", "SystemConfig") # capital letter corrected # except "STEP_ID", "appLog" stepInstance1.setValue("appName", "DaVinci") stepInstance1.setValue("appType", "root") stepInstance1.setValue("outputData", "@{PRODUCTION_ID}_@{JOB_ID}_@{STEP_NUMBER}.@{appType}") stepInstance1.setValue("optionsFile", "DVOfficialStrippingFile.opts") stepInstance1.setValue("optionsLine", opt_dav)
class WorkflowXMLHandler(ContentHandler): def __init__(self, new_wf=None): """ If new_wf defined, it will be used as root of document """ # this is an attribute for the object to be created from the XML document self.root=new_wf # the reference on the all document self.stack=None # to keep last object self.strings=None # to accumulate string object (list of strings) used to split long string def startDocument(self): #reset the process #self.root=None self.stack=[] self.strings=[] def endDocument(self): pass def startElement(self, name, attrs): #print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames() self.clearCharacters() # clear to remove empty or nonprintable characters if name == "Workflow": if self.root == None: #if root not defined by constractor self.root = Workflow() self.stack.append(self.root) elif name == "StepDefinition": obj = StepDefinition("TemporaryXMLObject_StepDefinition") if self.root == None: # in case we are saving Step only self.root = obj self.stack.append(obj) elif name == "StepInstance": obj = StepInstance("TemporaryXMLObject_StepInstance") self.stack.append(obj) elif name == "ModuleDefinition": obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition") if self.root == None: # in case we are saving Module only self.root = obj self.stack.append(obj) elif name == "ModuleInstance": obj = ModuleInstance("TemporaryXMLObject_ModuleInstance") self.stack.append(obj) elif name == "Parameter": obj = Parameter(str(attrs['name']), None, str(attrs['type']), str(attrs['linked_module']), str(attrs['linked_parameter']), str(attrs['in']), str(attrs['out']), str(attrs['description'])) self.stack.append(obj) # TEMPORARY CODE elif name=="origin" or name == "version" or name == "name" or name == "type" or name == "value" or\ name == "required" or name == "descr_short" or name == "name" or name == "type" or name == "description" or name == "body": pass else: print "UNTREATED! startElement name=", name, "attr=", attrs.getLength(), attrs.getNames() pass def endElement(self, name): #print name, "endElement" # attributes if name=="origin": self.stack[len(self.stack)-1].setOrigin(self.getCharacters()) elif name == "version": self.stack[len(self.stack)-1].setVersion(self.getCharacters()) elif name == "name": self.stack[len(self.stack)-1].setName(self.getCharacters()) elif name == "type": self.stack[len(self.stack)-1].setType(self.getCharacters()) elif name == "required": self.stack[len(self.stack)-1].setRequired(self.getCharacters()) elif name == "descr_short": self.stack[len(self.stack)-1].setDescrShort(self.getCharacters()) elif name == "name": self.stack[len(self.stack)-1].setName(self.getCharacters()) elif name == "type": self.stack[len(self.stack)-1].setType(self.getCharacters()) elif name == "description": self.stack[len(self.stack)-1].setDescription(self.getCharacters()) elif name == "body": self.stack[len(self.stack)-1].setBody(self.getCharacters()) elif name == "value": ch = self.getCharacters() # to keep compatibility with the old version # were """ was not used for the string if self.stack[len(self.stack)-1].isTypeString(): self.stack[len(self.stack)-1].setValue(ch) else: self.stack[len(self.stack)-1].setValue(eval(ch)) #objects elif name=="Workflow": self.stack.pop() elif name == "StepDefinition": self.root.step_definitions.append(self.stack.pop()) elif name == "StepInstance": self.root.step_instances.append(self.stack.pop()) elif name == "ModuleDefinition": self.root.addModule(self.stack.pop()) elif name == "ModuleInstance": obj=self.stack.pop() self.stack[len(self.stack)-1].module_instances.append(obj) elif name == "Parameter": obj=self.stack.pop(); self.stack[len(self.stack)-1].addParameter(obj) else: print "UNTREATED! endElement", name def getCharacters(self): # combine all strings and clear the list ret = ''.join(self.strings) self.clearCharacters() return str(ret) def clearCharacters(self): del self.strings self.strings=[] def characters(self, content): self.strings.append(content)
opt_brunel = opt_brunel+";DstWriter.Output = \"DATAFILE=\'PFN:@{outputData}\' TYP=\'POOL_ROOTTREE\' OPT=\'RECREATE\'\"" opt_brunel = opt_brunel+";EvtTupleSvc.Output = {\"EVTTAGS2 DATAFILE=\'PFN:@{etcf}\' TYP=\'POOL_ROOTTREE\' OPT=\'RECREATE\'\"}" #opt_brunel = opt_brunel+";EventSelector.Input = {\"COLLECTION=\'TagCreator/1\' DATAFILE=\'@{InputData}\' TYPE=\'POOL_ROOTTREE\' SEL=\'(GlobalOr>=1)\' OPT=\'READ\'\"}" opt_brunel = opt_brunel+";EventPersistencySvc.CnvServices += { \"LHCb::RawDataCnvSvc\" }" opt_brunel = opt_brunel+";ApplicationMgr.TopAlg += {\"StoreExplorerAlg\"}" opt_brunel = opt_brunel+";StoreExplorerAlg.Load = 1" opt_brunel = opt_brunel+";StoreExplorerAlg.PrintFreq = 0.99" opt_brunel = opt_brunel+";IODataManager.AgeLimit = 2" step3 = StepDefinition('Job_Finalization') step3.addModule(module6) moduleInstance6 = step3.createModuleInstance('JobFinalization','module6') ############## WORKFLOW ################################# workflow1 = Workflow(name='CCRC-joel-test') workflow1.setDescription('Workflow of GaudiApplication') workflow1.addStep(step1) step1_prefix="step1_" stepInstance1 = workflow1.createStepInstance('Gaudi_App_Step', 'Step1') # lets link all parameters them up with the level of workflow stepInstance1.setLink("systemConfig","self", "SystemConfig") # capital letter corrected # except "STEP_ID", "appLog" stepInstance1.setValue("appName", "DaVinci") stepInstance1.setValue("appType", "root") stepInstance1.setValue("outputData","@{PRODUCTION_ID}_@{JOB_ID}_@{STEP_NUMBER}.@{appType}") stepInstance1.setValue("optionsFile", "DVOfficialStrippingFile.opts") stepInstance1.setValue("optionsLine",opt_dav) stepInstance1.setValue("outputDataSE","Tier1_M-DST") stepInstance1.setLink("inputData","self", "InputData") # KGG linked with InputData of the Workflow
class Job: ############################################################################# def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ): """Instantiates the Workflow object and some default parameters. """ self.log = gLogger self.section = COMPONENT_NAME self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True #gConfig.getValue('Tier0SE-tape','SEName') self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo( disableVOMS = True ) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup( ret['Value']['group'] ) self.group = vo self.site = 'ANY' #ANY #self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] self.systemConfig = 'ANY' self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE', 'MinCPUTime': 'other.NAME<=VALUE', 'Site': 'other.NAME=="VALUE"', 'Platform': 'other.NAME=="VALUE"', #'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately 'BannedSites': 'other.Site!="VALUE"', 'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'} ##Add member to handle Parametric jobs self.parametric = {} self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow( script ) #Global error dictionary self.errorDict = {} ############################################################################# def setExecutable( self, executable, arguments = '', logFile = '' ): """Helper function. Specify executable script to run with optional arguments and log file for standard output. These can be either: - Submission of a python or shell script to DIRAC - Can be inline scripts e.g. C{'/bin/ls'} - Scripts as executables e.g. python or shell script file Example usage: >>> job = Job() >>> job.setExecutable('myScript.py') @param executable: Executable @type executable: string @param arguments: Optional arguments to executable @type arguments: string @param logFile: Optional log file name @type logFile: string """ kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile} if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or \ not type( logFile ) == type( ' ' ): return self._reportError( 'Expected strings for executable and arguments', **kwargs ) if os.path.exists( executable ): self.log.verbose( 'Found script executable file %s' % ( executable ) ) self.addToInputSandbox.append( executable ) logName = '%s.log' % ( os.path.basename( executable ) ) moduleName = os.path.basename( executable ) else: self.log.verbose( 'Found executable code' ) logName = 'CodeOutput.log' moduleName = 'CodeSegment' if logFile: if type( logFile ) == type( ' ' ): logName = logFile self.stepCount += 1 moduleName = moduleName.replace( '.', '' ) stepNumber = self.stepCount stepDefn = 'ScriptStep%s' % ( stepNumber ) step = self.__getScriptStep( stepDefn ) stepName = 'RunScriptStep%s' % ( stepNumber ) logPrefix = 'Script%s_' % ( stepNumber ) logName = '%s%s' % ( logPrefix, logName ) self.addToOutputSandbox.append( logName ) self.workflow.addStep( step ) # Define Step and its variables stepInstance = self.workflow.createStepInstance( stepDefn, stepName ) stepInstance.setValue( "name", moduleName ) stepInstance.setValue( "logFile", logName ) stepInstance.setValue( "executable", executable ) if arguments: stepInstance.setValue( "arguments", arguments ) return S_OK() ############################################################################# def setName( self, jobName ): """Helper function. A name for the job can be specified if desired. This will appear in the JobName field of the monitoring webpage. If nothing is specified a default value will appear. Example usage: >>> job=Job() >>> job.setName("myJobName") @param jobName: Name of job @type jobName: string """ kwargs = {'jobname':jobName} if not type( jobName ) == type( ' ' ): return self._reportError( 'Expected strings for job name', **kwargs ) else: self.workflow.setName( jobName ) self._addParameter( self.workflow, 'JobName', 'JDL', jobName, 'User specified name' ) return S_OK() ############################################################################# def setInputSandbox( self, files ): """Helper function. Specify input sandbox files less than 10MB in size. If over 10MB, files or a directory may be uploaded to Grid storage, see C{dirac.uploadSandbox()}. Paths to the options file and (if required) 'lib/' directory of the DLLs are specified here. Default is local directory. CMT requirements files or executables may be placed in the lib/ directory if desired. The lib/ directory is transferred to the Grid Worker Node before the job executes. Files / directories can be specified using the '*' character e.g. *.txt these are resolved correctly before job execution on the WN. Example usage: >>> job = Job() >>> job.setInputSandbox(['DaVinci.opts']) @param files: Input sandbox files, can specify full path @type files: Single string or list of strings ['',''] """ if type( files ) == list and len( files ): resolvedFiles = self._resolveInputSandbox( files ) fileList = ';'.join( resolvedFiles ) description = 'Input sandbox file list' self._addParameter( self.workflow, 'InputSandbox', 'JDL', fileList, description ) #self.sandboxFiles=resolvedFiles elif type( files ) == type( " " ): resolvedFiles = self._resolveInputSandbox( [files] ) fileList = ';'.join( resolvedFiles ) description = 'Input sandbox file' #self.sandboxFiles = [files] self._addParameter( self.workflow, 'InputSandbox', 'JDL', fileList, description ) else: kwargs = {'files':files} return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs ) return S_OK() ############################################################################# def setParametricInputSandbox( self, files ): """Helper function. Specify input sandbox files to used as parameters in the Parametric jobs. The possibilities are identical to the setInputSandbox. Example usage: >>> job = Job() >>> job.setParametricInputSandbox(['LFN:/Some_file','LFN:/Some_other_file']) @param files: Logical File Names @type files: Single LFN string or list of LFNs """ kwargs = {'files':files} if type( files ) == list and len( files ): for fileName in files: if not fileName.lower().count( "lfn:" ): return self._reportError( 'All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( files ) fileList = ";".join( resolvedFiles ) self.parametric['InputSandbox'] = fileList #self.sandboxFiles=resolvedFiles elif type( files ) == type( " " ): if not files.lower().count( "lfn:" ): return self._reportError( 'All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( [files] ) fileList = ";".join( resolvedFiles ) self.parametric['InputSandbox'] = fileList #self.sandboxFiles = [files] else: return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs ) return S_OK() ############################################################################# def setOutputSandbox( self, files ): """Helper function. Specify output sandbox files. If specified files are over 10MB, these may be uploaded to Grid storage with a notification returned in the output sandbox. Example usage: >>> job = Job() >>> job.setOutputSandbox(['DaVinci_v19r12.log','DVNTuples.root']) @param files: Output sandbox files @type files: Single string or list of strings ['',''] """ if type( files ) == list and len( files ): fileList = ';'.join( files ) description = 'Output sandbox file list' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description ) elif type( files ) == type( " " ): description = 'Output sandbox file' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description ) else: kwargs = {'files':files} return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs ) return S_OK() ############################################################################# def setInputData( self, lfns ): """Helper function. Specify input data by Logical File Name (LFN). Example usage: >>> job = Job() >>> job.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) @param lfns: Logical File Names @type lfns: Single LFN string or list of LFNs """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): lfns[i] = lfns[i].replace( 'LFN:', '' ) inputData = map( lambda x: 'LFN:' + x, lfns ) inputDataStr = ';'.join( inputData ) description = 'List of input data specified by LFNs' self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description ) elif type( lfns ) == type( ' ' ): #single LFN description = 'Input data specified by LFN' self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description ) else: kwargs = {'lfns':lfns} return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs ) return S_OK() ############################################################################# def setParametricInputData( self, lfns ): """Helper function. Specify input data by Logical File Name (LFN) to be used as a parameter in a parametric job Example usage: >>> job = Job() >>> job.setParametricInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) @param lfns: Logical File Names @type lfns: Single LFN string or list of LFNs """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): lfns[i] = lfns[i].replace( 'LFN:', '' ) inputData = map( lambda x: 'LFN:' + x, lfns ) inputDataStr = ';'.join( inputData ) self.parametric['InputData'] = inputDataStr elif type( lfns ) == type( ' ' ): #single LFN self.parametric['InputData'] = lfns else: kwargs = {'lfns':lfns} return self._reportError( 'Expected lfn string or list of lfns for parametric input data', **kwargs ) return S_OK() ############################################################################# def setInputDataPolicy( self, policy, dataScheduling = True ): """Helper function. Specify a job input data policy, this takes precedence over any site specific or global settings. Possible values for policy are 'Download' or 'Protocol' (case-insensitive). This requires that the module locations are defined for the VO in the CS. Example usage: >>> job = Job() >>> job.setInputDataPolicy('download') """ kwargs = {'policy':policy, 'dataScheduling':dataScheduling} csSection = '/Operations/InputDataPolicy' possible = ['Download', 'Protocol'] finalPolicy = '' for value in possible: if policy.lower() == value.lower(): finalPolicy = value if not finalPolicy: return self._reportError( 'Expected one of %s for input data policy' % ( ', '.join( possible ) ), __name__, **kwargs ) jobPolicy = gConfig.getValue( '%s/%s' % ( csSection, finalPolicy ), '' ) if not jobPolicy: return self._reportError( 'Could not get value for CS option %s/%s' % ( csSection, finalPolicy ), __name__, **kwargs ) description = 'User specified input data policy' self._addParameter( self.workflow, 'InputDataPolicy', 'JDL', jobPolicy, description ) if not dataScheduling and policy.lower() == 'download': self.log.verbose( 'Scheduling by input data is disabled, jobs will run anywhere and download input data' ) self._addParameter( self.workflow, 'DisableDataScheduling', 'JDL', 'True', 'Disable scheduling by input data' ) if not dataScheduling and policy.lower() != 'download': self.log.error( 'Expected policy to be "download" for bypassing data scheduling' ) return self._reportError( 'Expected policy to be "download" for bypassing data scheduling', __name__, **kwargs ) return S_OK() ############################################################################# def setOutputData( self, lfns, outputSE = None, outputPath = '' ): """Helper function. For specifying output data to be registered in Grid storage. If a list of OutputSEs are specified the job wrapper will try each in turn until successful. If the OutputPath is specified this will appear only after / <VO> / user / <initial> / <username> directory. Example usage: >>> job = Job() >>> job.setOutputData(['DVNtuple.root']) @param lfns: Output data file or files @type lfns: Single string or list of strings ['',''] @param outputSE: Optional parameter to specify the Storage Element @param outputPath: Optional parameter to specify part of the path in the storage (see above) Element to store data or files, e.g. CERN-tape @type outputSE: string or list @type outputPath: string """ if outputSE == None: outputSE = [] kwargs = {'lfns':lfns, 'OutputSE':outputSE, 'OutputPath':outputPath} if type( lfns ) == list and len( lfns ): outputDataStr = ';'.join( lfns ) description = 'List of output data files' self._addParameter( self.workflow, 'OutputData', 'JDL', outputDataStr, description ) elif type( lfns ) == type( " " ): description = 'Output data file' self._addParameter( self.workflow, 'OutputData', 'JDL', lfns, description ) else: return self._reportError( 'Expected file name string or list of file names for output data', **kwargs ) if outputSE: description = 'User specified Output SE' if type( outputSE ) in types.StringTypes: outputSE = [outputSE] elif type( outputSE ) != types.ListType: return self._reportError( 'Expected string or list for OutputSE', **kwargs ) outputSE = ';'.join( outputSE ) self._addParameter( self.workflow, 'OutputSE', 'JDL', outputSE, description ) if outputPath: description = 'User specified Output Path' if not type( outputPath ) in types.StringTypes: return self._reportError( 'Expected string for OutputPath', **kwargs ) # Remove leading "/" that might cause problems with os.path.join # FIXME: this will prevent to set OutputPath outside the Home of the User while outputPath[0] == '/': outputPath = outputPath[1:] self._addParameter( self.workflow, 'OutputPath', 'JDL', outputPath, description ) return S_OK() ############################################################################# def setPlatform( self, backend ): """Developer function. Choose submission pool on which job is executed e.g. DIRAC, LCG. Default in place for users. """ #should add protection here for list of supported platforms kwargs = {'backend':backend} if not type( backend ) == type( " " ): return self._reportError( 'Expected string for platform', **kwargs ) if not backend.lower() == 'any': self._addParameter( self.workflow, 'SubmitPools', 'JDL', backend, 'Platform type' ) return S_OK() ############################################################################# def setSystemConfig( self, config ): """Helper function. Choose system configuration (e.g. where user DLLs have been compiled). Default ANY in place for user jobs. Available system configurations can be browsed via dirac.checkSupportedPlatforms() method. Example usage: >>> job=Job() >>> job.setSystemConfig("slc4_ia32_gcc34") @param config: architecture, CMTCONFIG value @type config: string """ kwargs = {'config':config} if not type( config ) == type( " " ): return self._reportError( 'Expected string for system configuration', **kwargs ) description = 'User specified system configuration for job' self._addParameter( self.workflow, 'SystemConfig', 'JDLReqt', config, description ) self.systemConfig = config return S_OK() ############################################################################# def setCPUTime( self, timeInSecs ): """Helper function. Under Development. Specify CPU time requirement in DIRAC units. Example usage: >>> job = Job() >>> job.setCPUTime(5000) @param timeInSecs: CPU time @type timeInSecs: Int """ kwargs = {'timeInSecs':timeInSecs} if not type( timeInSecs ) == int: try: timeInSecs = int( timeInSecs ) except Exception: if not re.search( '{{', timeInSecs ): return self._reportError( 'Expected numerical string or int for CPU time in seconds', **kwargs ) description = 'CPU time in secs' self._addParameter( self.workflow, 'MaxCPUTime', 'JDLReqt', timeInSecs, description ) return S_OK() ############################################################################# def setDestination( self, destination ): """Helper function. Can specify a desired destination site or sites for job. This can be useful for debugging purposes but often limits the possible candidate sites and overall system response time. Example usage: >>> job = Job() >>> job.setDestination('LCG.CERN.ch') @param destination: site string @type destination: string or list """ kwargs = {'destination':destination} if type( destination ) == type( " " ): if not re.search( '^DIRAC.', destination ) and not destination.lower() == 'any': result = self.__checkSiteIsValid( destination ) if not result['OK']: return self._reportError( '%s is not a valid destination site' % ( destination ), **kwargs ) description = 'User specified destination site' self._addParameter( self.workflow, 'Site', 'JDLReqt', destination, description ) elif type( destination ) == list: for site in destination: if not re.search( '^DIRAC.', site ) and not site.lower() == 'any': result = self.__checkSiteIsValid( site ) if not result['OK']: return self._reportError( '%s is not a valid destination site' % ( destination ), **kwargs ) destSites = ';'.join( destination ) description = 'List of sites selected by user' self._addParameter( self.workflow, 'Site', 'JDLReqt', destSites, description ) else: return self._reportError( '%s is not a valid destination site, expected string' % ( destination ), **kwargs ) return S_OK() ############################################################################# def __checkSiteIsValid( self, site ): """Internal function to check that a site name is valid. """ sites = getSiteCEMapping() if not sites['OK']: return S_ERROR( 'Could not get site CE mapping' ) siteList = sites['Value'].keys() if not site in siteList: return S_ERROR( 'Specified site %s is not in list of defined sites' % site ) return S_OK( '%s is valid' % site ) ############################################################################# def setDestinationCE( self, ceName ): """ Developer function. Allows to direct a job to a particular Grid CE. """ kwargs = {'ceName':ceName} diracSite = getSiteForCE( ceName ) if not diracSite['OK']: return self._reportError( diracSite['Message'], **kwargs ) if not diracSite['Value']: return self._reportError( 'No DIRAC site name found for CE %s' % ( ceName ), **kwargs ) diracSite = diracSite['Value'] self.setDestination( diracSite ) self._addJDLParameter( 'GridRequiredCEs', ceName ) return S_OK() ############################################################################# def setBannedSites( self, sites ): """Helper function. Can specify a desired destination site for job. This can be useful for debugging purposes but often limits the possible candidate sites and overall system response time. Example usage: >>> job = Job() >>> job.setBannedSites(['LCG.GRIDKA.de','LCG.CNAF.it']) @param sites: single site string or list @type sites: string or list """ if type( sites ) == list and len( sites ): bannedSites = ';'.join( sites ) description = 'List of sites excluded by user' self._addParameter( self.workflow, 'BannedSites', 'JDLReqt', bannedSites, description ) elif type( sites ) == type( " " ): description = 'Site excluded by user' self._addParameter( self.workflow, 'BannedSites', 'JDLReqt', sites, description ) else: kwargs = {'sites':sites} return self._reportError( 'Expected site string or list of sites', **kwargs ) return S_OK() ############################################################################# def setOwner( self, ownerProvided ): """Developer function. Normally users should always specify their immutable DIRAC nickname. """ if not type( ownerProvided ) == type( " " ): return self._reportError( 'Expected string for owner', **{'ownerProvided':ownerProvided} ) self._addParameter( self.workflow, 'Owner', 'JDL', ownerProvided, 'User specified ID' ) return S_OK() ############################################################################# def setOwnerGroup( self, ownerGroup ): """Developer function. Allows to force expected owner group of proxy. """ if not type( ownerGroup ) == type( " " ): return self._reportError( 'Expected string for job owner group', **{'ownerGroup':ownerGroup} ) self._addParameter( self.workflow, 'OwnerGroup', 'JDL', ownerGroup, 'User specified owner group.' ) return S_OK() ############################################################################# def setType( self, jobType ): """Developer function. Specify job type for testing purposes. """ if not type( jobType ) == type( " " ): return self._reportError( 'Expected string for job type', **{'jobType':jobType} ) self._addParameter( self.workflow, 'JobType', 'JDL', jobType, 'User specified type' ) self.type = jobType return S_OK() ############################################################################# def _setSoftwareTags( self, tags ): """Developer function. Choose any software tags if desired. These are not compulsory but will ensure jobs only arrive at an LCG site where the software is preinstalled. Without the tags, missing software is installed automatically by the Job Agent. Example usage: >>> job=Job() >>> job.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10','VO-lhcb-Gauss-v25r12']) @param tags: software tags @type tags: string or list """ if type( tags ) == type( " " ): self._addParameter( self.workflow, 'SoftwareTag', 'JDL', tags, 'VO software tag' ) elif type( tags ) == list: swTags = ';'.join( tags ) self._addParameter( self.workflow, 'SoftwareTag', 'JDL', swTags, 'List of VO software tags' ) else: kwargs = {'tags':tags} return self._reportError( 'Expected String or List of software tags', **kwargs ) return S_OK() ############################################################################# def setJobGroup( self, jobGroup ): """Helper function. Allows to group certain jobs according to an ID. Example usage: >>> job = Job() >>> job.setJobGroup('Bs2JPsiPhi') @param jobGroup: JobGroup name @type jobGroup: string """ if not type( jobGroup ) == type( " " ): return self._reportError( 'Expected string for job group name', **{'jobGroup':jobGroup} ) description = 'User specified job group' self._addParameter( self.workflow, 'JobGroup', 'JDL', jobGroup, description ) return S_OK() ############################################################################# def setLogLevel( self, logLevel ): """Helper function. Optionally specify a DIRAC logging level for the job, e.g. ALWAYS, INFO, VERBOSE, WARN, DEBUG by default this is set to the info level. Example usage: >>> job = Job() >>> job.setLogLevel('debug') @param logLevel: Logging level @type logLevel: string """ kwargs = {'logLevel':logLevel} if type( logLevel ) in types.StringTypes: if logLevel.upper() in gLogger._logLevels.getLevels(): description = 'User specified logging level' self.logLevel = logLevel self._addParameter( self.workflow, 'LogLevel', 'JDL', logLevel, description ) else: return self._reportError( 'Error Level "%s" not valid' % logLevel, **kwargs ) else: return self._reportError( 'Expected string for logging level', **kwargs ) return S_OK() ############################################################################# def setConfigArgs( self, cfgString ): """Developer function. Allow to pass arbitrary settings to the payload configuration service environment. """ if not type( cfgString ) == type( " " ): return self._reportError( 'Expected string for DIRAC Job Config Args', **{'cfgString':cfgString} ) description = 'User specified cfg settings' self._addParameter( self.workflow, 'JobConfigArgs', 'JDL', cfgString, description ) return S_OK() ############################################################################# def setMode( self, mode ): """Developer function. Under development. """ if not type( mode ) == type( " " ): return self._reportError( 'Expected string for job mode', **{'mode':mode} ) description = 'Choose a different DIRAC job mode' self._addParameter( self.workflow, 'JobMode', 'JDL', mode, description ) return S_OK() ############################################################################# def selectSetup( self, setup ): """Developer function. Under development. """ if not type( setup ) == type( " " ): return self._reportError( 'Expected string for DIRAC setup', **{'setup':setup} ) description = 'Choose a different DIRAC setup in which to execute the job' self._addParameter( self.workflow, 'DIRACSetup', 'JDL', setup, description ) return S_OK() ############################################################################# def setExecutionEnv( self, environmentDict ): """Helper function. Optionally specify a dictionary of key, value pairs to be set before the job executes e.g. {'MYVAR':3} The standard application environment variables are always set so this is intended for user variables only. Example usage: >>> job = Job() >>> job.setExecutionEnviroment({'<MYVARIABLE>':'<VALUE>'}) @param environmentDict: Environment variables @type environmentDict: dictionary """ kwargs = {'environmentDict':environmentDict} if not type( environmentDict ) == type( {} ): return self._reportError( 'Expected dictionary of environment variables', **kwargs ) environment = [] for var, val in environmentDict.items(): try: environment.append( '='.join( [str( var ), str( val )] ) ) except Exception: return self._reportError( 'Expected string for environment variable key value pairs', **kwargs ) envStr = ';'.join( environment ) description = 'Env vars specified by user' self._addParameter( self.workflow, 'ExecutionEnvironment', 'JDL', envStr, description ) return S_OK() ############################################################################# def sendMail( self ): """Under development. """ description = 'Optional flag to send email when jobs complete' self._addParameter( self.workflow, 'SendMail', 'JDL', 'True', description ) ############################################################################# def createCode( self ): """Developer function. Wrapper method to create the code. """ return self.workflow.createCode() ############################################################################# def execute( self ): """Developer function. Executes the job locally. """ self.createCode() # code = self.createCode() # eval(compile(code,'<string>','exec')) self.workflow.execute() ############################################################################# def _getParameters( self ): """Developer function. Method to return the workflow parameters. """ wfParams = {} params = self.workflow.parameters for par in params: wfParams[par.getName()] = par.getValue() return wfParams ############################################################################# def _dumpParameters( self, showType = None ): """Developer function. Method to print the workflow parameters. """ paramsDict = {} paramList = self.workflow.parameters for param in paramList: paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()} self.log.info( '--------------------------------------' ) self.log.info( 'Workflow parameter summary: ' ) self.log.info( '--------------------------------------' ) #print self.workflow.parameters #print params.getParametersNames() for name, props in paramsDict.items(): ptype = paramsDict[name]['type'] value = paramsDict[name]['value'] if showType: if ptype == showType: self.log.info( 'NAME: %s\nTYPE: %s\nVALUE: %s ' % ( name, ptype, value ) ) self.log.info( '--------------------------------------' ) else: self.log.info( 'NAME: %s\nTYPE: %s\nVALUE: %s ' % ( name, ptype, value ) ) self.log.info( '--------------------------------------' ) ############################################################################# def __setJobDefaults( self ): """Set job default values. Note that the system configuration is set to "ANY". """ self._addParameter( self.workflow, 'JobType', 'JDL', self.type, 'Job Type' ) self._addParameter( self.workflow, 'Priority', 'JDL', self.priority, 'User Job Priority' ) self._addParameter( self.workflow, 'JobGroup', 'JDL', self.group, 'Name of the JobGroup' ) self._addParameter( self.workflow, 'JobName', 'JDL', self.name, 'Name of Job' ) #self._addParameter(self.workflow,'DIRACSetup','JDL',self.setup,'DIRAC Setup') self._addParameter( self.workflow, 'SystemConfig', 'JDLReqt', self.systemConfig, 'System configuration for job' ) self._addParameter( self.workflow, 'Site', 'JDL', self.site, 'Site Requirement' ) self._addParameter( self.workflow, 'Origin', 'JDL', self.origin, 'Origin of client' ) self._addParameter( self.workflow, 'StdOutput', 'JDL', self.stdout, 'Standard output file' ) self._addParameter( self.workflow, 'StdError', 'JDL', self.stderr, 'Standard error file' ) self._addParameter( self.workflow, 'InputData', 'JDL', '', 'Default null input data value' ) self._addParameter( self.workflow, 'LogLevel', 'JDL', self.logLevel, 'Job Logging Level' ) #Those 2 below are need for on-site resolution self._addParameter( self.workflow, 'ParametricInputData', 'JDL', '', 'Default null parametric input data value' ) self._addParameter( self.workflow, 'ParametricInputSandbox', 'JDL', '', 'Default null parametric input sandbox value' ) ############################################################################# def _addParameter( self, wObject, name, ptype, value, description, io = 'input' ): """ Internal Function Adds a parameter to the object. """ if io == 'input': inBool = True outBool = False elif io == 'output': inBool = False outBool = True else: raise TypeError, 'I/O flag is either input or output' par = Parameter( name, value, ptype, "", "", inBool, outBool, description ) wObject.addParameter( Parameter( parameter = par ) ) return par ############################################################################ def _resolveInputSandbox( self, inputSandbox ): """ Internal function. Resolves wildcards for input sandbox files. This is currently linux specific and should be modified. """ resolvedIS = [] for i in inputSandbox: if not re.search( '\*', i ): if not os.path.isdir( i ): resolvedIS.append( i ) for name in inputSandbox: if re.search( '\*', name ): #escape the star character... cmd = 'ls -d ' + name output = shellCall( 10, cmd ) if not output['OK']: self.log.error( 'Could not perform: %s' % ( cmd ) ) else: files = output['Value'].split() for check in files: if os.path.isfile( check ): self.log.verbose( 'Found file ' + check + ' appending to Input Sandbox' ) resolvedIS.append( check ) if os.path.isdir( check ): if re.search( '/$', check ): #users can specify e.g. /my/dir/lib/ check = check[:-1] tarname = os.path.basename( check ) directory = os.path.dirname( check ) #if just the directory this is null if directory: cmd = 'tar cfz ' + tarname + '.tar.gz ' + ' -C ' + directory + ' ' + tarname else: cmd = 'tar cfz ' + tarname + '.tar.gz ' + tarname output = shellCall( 60, cmd ) if not output['OK']: self.log.error( 'Could not perform: %s' % ( cmd ) ) resolvedIS.append( tarname + '.tar.gz' ) self.log.verbose( 'Found directory ' + check + ', appending ' + check + '.tar.gz to Input Sandbox' ) if os.path.isdir( name ): self.log.verbose( 'Found specified directory ' + name + ', appending ' + name + '.tar.gz to Input Sandbox' ) if re.search( '/$', name ): #users can specify e.g. /my/dir/lib/ name = name[:-1] tarname = os.path.basename( name ) directory = os.path.dirname( name ) #if just the directory this is null if directory: cmd = 'tar cfz ' + tarname + '.tar.gz ' + ' -C ' + directory + ' ' + tarname else: cmd = 'tar cfz ' + tarname + '.tar.gz ' + tarname output = shellCall( 60, cmd ) if not output['OK']: self.log.error( 'Could not perform: %s' % ( cmd ) ) else: resolvedIS.append( tarname + '.tar.gz' ) return resolvedIS ############################################################################# def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step ############################################################################# def _toXML( self ): """Creates an XML representation of itself as a Job. """ return self.workflow.toXML() ############################################################################# def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir """Creates a JDL representation of itself as a Job. """ #Check if we have to do old bootstrap... classadJob = ClassAd( '[]' ) paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()} scriptname = 'jobDescription.xml' arguments = [] if self.script: if os.path.exists( self.script ): scriptname = os.path.abspath( self.script ) self.log.verbose( 'Found script name %s' % scriptname ) else: if xmlFile: self.log.verbose( 'Found XML File %s' % xmlFile ) scriptname = xmlFile arguments.append( os.path.basename( scriptname ) ) self.addToInputSandbox.append( scriptname ) if paramsDict.has_key( 'LogLevel' ): if paramsDict['LogLevel']['value']: arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) ) else: self.log.warn( 'Job LogLevel defined with null value' ) if paramsDict.has_key( 'DIRACSetup' ): if paramsDict['DIRACSetup']['value']: arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) ) else: self.log.warn( 'Job DIRACSetup defined with null value' ) if paramsDict.has_key( 'JobMode' ): if paramsDict['JobMode']['value']: arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) ) else: self.log.warn( 'Job Mode defined with null value' ) if paramsDict.has_key( 'JobConfigArgs' ): if paramsDict['JobConfigArgs']['value']: arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) ) else: self.log.warn( 'JobConfigArgs defined with null value' ) classadJob.insertAttributeString( 'Executable', self.executable ) self.addToOutputSandbox.append( self.stderr ) self.addToOutputSandbox.append( self.stdout ) #Extract i/o sandbox parameters from steps and any input data parameters #to do when introducing step-level api... #To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ';'.join( self.addToInputSandbox ) if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] finalInputSandbox = currentFiles + ';' + extraFiles uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) ) paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox ) self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) ) else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = extraFiles paramsDict['InputSandbox']['type'] = 'JDL' if self.addToOutputSandbox: extraFiles = ';'.join( self.addToOutputSandbox ) if paramsDict.has_key( 'OutputSandbox' ): currentFiles = paramsDict['OutputSandbox']['value'] finalOutputSandbox = currentFiles + ';' + extraFiles uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) ) paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox ) self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) ) else: paramsDict['OutputSandbox'] = {} paramsDict['OutputSandbox']['value'] = extraFiles paramsDict['OutputSandbox']['type'] = 'JDL' if self.addToInputData: extraFiles = ';'.join( self.addToInputData ) if paramsDict.has_key( 'InputData' ): currentFiles = paramsDict['InputData']['value'] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ';' + extraFiles uniqueInputData = uniqueElements( finalInputData.split( ';' ) ) paramsDict['InputData']['value'] = ';'.join( uniqueInputData ) self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) ) else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = extraFiles paramsDict['InputData']['type'] = 'JDL' ###Here handle the Parametric values if self.parametric: if self.parametric.has_key( 'InputData' ): if paramsDict.has_key( 'InputData' ): if paramsDict['InputData']['value']: currentFiles = paramsDict['InputData']['value'] + ";%s" paramsDict['InputData']['value'] = currentFiles else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = "%s" paramsDict['InputData']['type'] = 'JDL' self.parametric['files'] = self.parametric['InputData'] arguments.append( ' -p ParametricInputData=%s' ) elif self.parametric.has_key( 'InputSandbox' ): if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] + ";%s" paramsDict['InputSandbox']['value'] = currentFiles else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = '%s' paramsDict['InputSandbox']['type'] = 'JDL' self.parametric['files'] = self.parametric['InputSandbox'] arguments.append( ' -p ParametricInputSandbox=%s' ) if self.parametric.has_key( 'files' ): paramsDict['Parameters'] = {} paramsDict['Parameters']['value'] = self.parametric['files'] paramsDict['Parameters']['type'] = 'JDL' ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) ) #Add any JDL parameters to classad obeying lists with ';' rule requirements = False for name, props in paramsDict.items(): ptype = props['type'] value = props['value'] if name.lower() == 'requirements' and ptype == 'JDL': self.log.verbose( 'Found existing requirements: %s' % ( value ) ) requirements = True if re.search( '^JDL', ptype ): if not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... classadJob.insertAttributeString( name, value ) else: classadJob.insertAttributeVectorString( name, ';'.split( value ) ) if not requirements: reqtsDict = self.reqParams exprn = '' plus = '' for name, props in paramsDict.items(): ptype = paramsDict[name]['type'] value = paramsDict[name]['value'] if not ptype == 'dict': if ptype == 'JDLReqt': if value and not value.lower() == 'any': plus = ' && ' if re.search( ';', value ): for val in value.split( ';' ): exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus else: exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus if len( plus ): exprn = exprn[:-len( plus )] if not exprn: exprn = 'true' self.log.verbose( 'Requirements: %s' % ( exprn ) ) #classadJob.set_expression('Requirements', exprn) self.addToInputSandbox.remove( scriptname ) self.addToOutputSandbox.remove( self.stdout ) self.addToOutputSandbox.remove( self.stderr ) jdl = classadJob.asJDL() start = jdl.find( '[' ) end = jdl.rfind( ']' ) return jdl[( start + 1 ):( end - 1 )] ############################################################################# def _setParamValue( self, name, value ): """Internal Function. Sets a parameter value, used for production. """ return self.workflow.setValue( name, value ) ############################################################################# def _addJDLParameter( self, name, value ): """Developer function, add an arbitrary JDL parameter. """ self._addParameter( self.workflow, name, 'JDL', value, 'Optional JDL parameter added' ) return self.workflow.setValue( name, value ) ############################################################################# def _getErrors( self ): """Returns the dictionary of stored errors that will prevent submission or execution. """ return self.errorDict ############################################################################# def _reportError( self, message, name = '', **kwargs ): """Internal Function. Gets caller method name and arguments, formats the information and adds an error to the global error dictionary to be returned to the user. """ className = name if not name: className = __name__ methodName = sys._getframe( 1 ).f_code.co_name arguments = [] for key in kwargs: if kwargs[key]: arguments.append( '%s = %s ( %s )' % ( key, kwargs[key], type( kwargs[key] ) ) ) finalReport = 'Problem with %s.%s() call:\nArguments: %s\nMessage: %s\n' % ( className, methodName, ', '.join( arguments ), message ) if self.errorDict.has_key( methodName ): tmp = self.errorDict[methodName] tmp.append( finalReport ) self.errorDict[methodName] = tmp else: self.errorDict[methodName] = [finalReport] self.log.verbose( finalReport ) return S_ERROR( finalReport )
class Job: ############################################################################# def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ): """Instantiates the Workflow object and some default parameters. """ self.log = gLogger self.section = COMPONENT_NAME self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True #gConfig.getValue('Tier0SE-tape','SEName') self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 self.group = getVO( 'lhcb' ) self.site = 'ANY' #ANY #self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] self.systemConfig = 'ANY' self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE', 'MinCPUTime': 'other.NAME<=VALUE', 'Site': 'other.NAME=="VALUE"', 'Platform': 'other.NAME=="VALUE"', #'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately 'BannedSites': 'other.Site!="VALUE"', 'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'} ##Add member to handle Parametric jobs self.parametric = {} self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow( script ) #Global error dictionary self.errorDict = {} ############################################################################# def setExecutable( self, executable, arguments = '', logFile = '' ): """Helper function. Specify executable script to run with optional arguments and log file for standard output. These can be either: - Submission of a python or shell script to DIRAC - Can be inline scripts e.g. C{'/bin/ls'} - Scripts as executables e.g. python or shell script file Example usage: >>> job = Job() >>> job.setExecutable('myScript.py') @param executable: Executable @type executable: string @param arguments: Optional arguments to executable @type arguments: string @param logFile: Optional log file name @type logFile: string """ kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile} if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or not type( logFile ) == type( ' ' ): return self._reportError( 'Expected strings for executable and arguments', **kwargs ) if os.path.exists( executable ): self.log.verbose( 'Found script executable file %s' % ( executable ) ) self.addToInputSandbox.append( executable ) logName = '%s.log' % ( os.path.basename( executable ) ) moduleName = os.path.basename( executable ) else: self.log.verbose( 'Found executable code' ) logName = 'CodeOutput.log' moduleName = 'CodeSegment' if logFile: if type( logFile ) == type( ' ' ): logName = logFile self.stepCount += 1 moduleName = moduleName.replace( '.', '' ) stepNumber = self.stepCount stepDefn = 'ScriptStep%s' % ( stepNumber ) step = self.__getScriptStep( stepDefn ) stepName = 'RunScriptStep%s' % ( stepNumber ) logPrefix = 'Script%s_' % ( stepNumber ) logName = '%s%s' % ( logPrefix, logName ) self.addToOutputSandbox.append( logName ) self.workflow.addStep( step ) # Define Step and its variables stepInstance = self.workflow.createStepInstance( stepDefn, stepName ) stepInstance.setValue( "name", moduleName ) stepInstance.setValue( "logFile", logName ) stepInstance.setValue( "executable", executable ) if arguments: stepInstance.setValue( "arguments", arguments ) return S_OK() ############################################################################# def setName( self, jobName ): """Helper function. A name for the job can be specified if desired. This will appear in the JobName field of the monitoring webpage. If nothing is specified a default value will appear. Example usage: >>> job=Job() >>> job.setName("myJobName") @param jobName: Name of job @type jobName: string """ kwargs = {'jobname':jobName} if not type( jobName ) == type( ' ' ): return self._reportError( 'Expected strings for job name', **kwargs ) else: self.workflow.setName( jobName ) self._addParameter( self.workflow, 'JobName', 'JDL', jobName, 'User specified name' ) return S_OK() ############################################################################# def setInputSandbox( self, files ): """Helper function. Specify input sandbox files less than 10MB in size. If over 10MB, files or a directory may be uploaded to Grid storage, see C{dirac.uploadSandbox()}. Paths to the options file and (if required) 'lib/' directory of the DLLs are specified here. Default is local directory. CMT requirements files or executables may be placed in the lib/ directory if desired. The lib/ directory is transferred to the Grid Worker Node before the job executes. Files / directories can be specified using the '*' character e.g. *.txt these are resolved correctly before job execution on the WN. Example usage: >>> job = Job() >>> job.setInputSandbox(['DaVinci.opts']) @param files: Input sandbox files, can specify full path @type files: Single string or list of strings ['',''] """ if type( files ) == list and len( files ): resolvedFiles = self._resolveInputSandbox( files ) fileList = string.join( resolvedFiles, ";" ) description = 'Input sandbox file list' self._addParameter( self.workflow, 'InputSandbox', 'JDL', fileList, description ) #self.sandboxFiles=resolvedFiles elif type( files ) == type( " " ): resolvedFiles = self._resolveInputSandbox( [files] ) fileList = string.join( resolvedFiles, ";" ) description = 'Input sandbox file' #self.sandboxFiles = [files] self._addParameter( self.workflow, 'InputSandbox', 'JDL', fileList, description ) else: kwargs = {'files':files} return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs ) return S_OK() ############################################################################# def setParametricInputSandbox( self, files ): """Helper function. Specify input sandbox files to used as parameters in the Parametric jobs. The possibilities are identical to the setInputSandbox. Example usage: >>> job = Job() >>> job.setParametricInputSandbox(['LFN:/Some_file','LFN:/Some_other_file']) @param files: Logical File Names @type files: Single LFN string or list of LFNs """ kwargs = {'files':files} if type( files ) == list and len( files ): for file in files: if not file.lower().count("lfn:"): return self._reportError('All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( files ) fileList = string.join( resolvedFiles, ";" ) self.parametric['InputSandbox']=fileList #self.sandboxFiles=resolvedFiles elif type( files ) == type( " " ): if not files.lower().count("lfn:"): return self._reportError('All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( [files] ) fileList = string.join( resolvedFiles, ";" ) self.parametric['InputSandbox']=fileList #self.sandboxFiles = [files] else: return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs ) return S_OK() ############################################################################# def setOutputSandbox( self, files ): """Helper function. Specify output sandbox files. If specified files are over 10MB, these may be uploaded to Grid storage with a notification returned in the output sandbox. Example usage: >>> job = Job() >>> job.setOutputSandbox(['DaVinci_v19r12.log','DVNTuples.root']) @param files: Output sandbox files @type files: Single string or list of strings ['',''] """ if type( files ) == list and len( files ): fileList = string.join( files, ";" ) description = 'Output sandbox file list' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description ) elif type( files ) == type( " " ): description = 'Output sandbox file' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description ) else: kwargs = {'files':files} return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs ) return S_OK() ############################################################################# def setInputData( self, lfns ): """Helper function. Specify input data by Logical File Name (LFN). Example usage: >>> job = Job() >>> job.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) @param lfns: Logical File Names @type lfns: Single LFN string or list of LFNs """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): lfns[i] = lfns[i].replace( 'LFN:', '' ) inputData = map( lambda x: 'LFN:' + x, lfns ) inputDataStr = string.join( inputData, ';' ) description = 'List of input data specified by LFNs' self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description ) elif type( lfns ) == type( ' ' ): #single LFN description = 'Input data specified by LFN' self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description ) else: kwargs = {'lfns':lfns} return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs ) return S_OK() ############################################################################# def setParametricInputData( self, lfns ): """Helper function. Specify input data by Logical File Name (LFN) to be used as a parameter in a parametric job Example usage: >>> job = Job() >>> job.setParametricInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) @param lfns: Logical File Names @type lfns: Single LFN string or list of LFNs """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): lfns[i] = lfns[i].replace( 'LFN:', '' ) inputData = map( lambda x: 'LFN:' + x, lfns ) inputDataStr = string.join( inputData, ';' ) self.parametric['InputData']=inputDataStr elif type( lfns ) == type( ' ' ): #single LFN self.parametric['InputData']=lfns else: kwargs = {'lfns':lfns} return self._reportError( 'Expected lfn string or list of lfns for parametric input data', **kwargs ) return S_OK() ############################################################################# def setInputDataPolicy( self, policy, dataScheduling = True ): """Helper function. Specify a job input data policy, this takes precedence over any site specific or global settings. Possible values for policy are 'Download' or 'Protocol' (case-insensitive). This requires that the module locations are defined for the VO in the CS. Example usage: >>> job = Job() >>> job.setInputDataPolicy('download') """ kwargs = {'policy':policy, 'dataScheduling':dataScheduling} csSection = '/Operations/InputDataPolicy' possible = ['Download', 'Protocol'] finalPolicy = '' for p in possible: if string.lower( policy ) == string.lower( p ): finalPolicy = p if not finalPolicy: return self._reportError( 'Expected one of %s for input data policy' % ( string.join( possible, ', ' ) ), __name__, **kwargs ) jobPolicy = gConfig.getValue( '%s/%s' % ( csSection, finalPolicy ), '' ) if not jobPolicy: return self._reportError( 'Could not get value for CS option %s/%s' % ( csSection, finalPolicy ), __name__, **kwargs ) description = 'User specified input data policy' self._addParameter( self.workflow, 'InputDataPolicy', 'JDL', jobPolicy, description ) if not dataScheduling and policy.lower() == 'download': self.log.verbose( 'Scheduling by input data is disabled, jobs will run anywhere and download input data' ) self._addParameter( self.workflow, 'DisableDataScheduling', 'JDL', 'True', 'Disable scheduling by input data' ) if not dataScheduling and policy.lower() != 'download': self.log.error( 'Expected policy to be "download" for bypassing data scheduling' ) return self._reportError( 'Expected policy to be "download" for bypassing data scheduling', __name__, **kwargs ) return S_OK() ############################################################################# def setOutputData( self, lfns, OutputSE = [], OutputPath = '' ): """Helper function. For specifying output data to be registered in Grid storage. If a list of OutputSEs are specified the job wrapper will try each in turn until successful. If the OutputPath is specified this will appear only after / <VO> / user / <initial> / <username> directory. Example usage: >>> job = Job() >>> job.setOutputData(['DVNtuple.root']) @param lfns: Output data file or files @type lfns: Single string or list of strings ['',''] @param OutputSE: Optional parameter to specify the Storage Element @param OutputPath: Optional parameter to specify part of the path in the storage (see above) Element to store data or files, e.g. CERN-tape @type OutputSE: string or list @type OutputPath: string """ kwargs = {'lfns':lfns, 'OutputSE':OutputSE, 'OutputPath':OutputPath} if type( lfns ) == list and len( lfns ): outputDataStr = string.join( lfns, ';' ) description = 'List of output data files' self._addParameter( self.workflow, 'OutputData', 'JDL', outputDataStr, description ) elif type( lfns ) == type( " " ): description = 'Output data file' self._addParameter( self.workflow, 'OutputData', 'JDL', lfns, description ) else: return self._reportError( 'Expected file name string or list of file names for output data', **kwargs ) if OutputSE: description = 'User specified Output SE' if type( OutputSE ) in types.StringTypes: OutputSE = [OutputSE] elif type( OutputSE ) != types.ListType: return self._reportError( 'Expected string or list for OutputSE', **kwargs ) OutputSE = ';'.join( OutputSE ) self._addParameter( self.workflow, 'OutputSE', 'JDL', OutputSE, description ) if OutputPath: description = 'User specified Output Path' if not type( OutputPath ) in types.StringTypes: return self._reportError( 'Expected string for OutputPath', **kwargs ) # Remove leading "/" that might cause problems with os.path.join while OutputPath[0] == '/': OutputPath = OutputPath[1:] self._addParameter( self.workflow, 'OutputPath', 'JDL', OutputPath, description ) return S_OK() ############################################################################# def setPlatform( self, backend ): """Developer function. Choose submission pool on which job is executed e.g. DIRAC, LCG. Default in place for users. """ #should add protection here for list of supported platforms kwargs = {'backend':backend} if not type( backend ) == type( " " ): return self._reportError( 'Expected string for platform', **kwargs ) if not backend.lower() == 'any': self._addParameter( self.workflow, 'SubmitPools', 'JDL', backend, 'Platform type' ) return S_OK() ############################################################################# def setSystemConfig( self, config ): """Helper function. Choose system configuration (e.g. where user DLLs have been compiled). Default ANY in place for user jobs. Available system configurations can be browsed via dirac.checkSupportedPlatforms() method. Example usage: >>> job=Job() >>> job.setSystemConfig("slc4_ia32_gcc34") @param config: architecture, CMTCONFIG value @type config: string """ kwargs = {'config':config} if not type( config ) == type( " " ): return self._reportError( 'Expected string for system configuration', **kwargs ) description = 'User specified system configuration for job' self._addParameter( self.workflow, 'SystemConfig', 'JDLReqt', config, description ) self.systemConfig = config return S_OK() ############################################################################# def setCPUTime( self, timeInSecs ): """Helper function. Under Development. Specify CPU time requirement in DIRAC units. Example usage: >>> job = Job() >>> job.setCPUTime(5000) @param timeInSecs: CPU time @type timeInSecs: Int """ kwargs = {'timeInSecs':timeInSecs} if not type( timeInSecs ) == int: try: timeInSecs = int( timeInSecs ) except Exception, x: if not re.search( '{{', timeInSecs ): return self._reportError( 'Expected numerical string or int for CPU time in seconds', **kwargs ) description = 'CPU time in secs' self._addParameter( self.workflow, 'MaxCPUTime', 'JDLReqt', timeInSecs, description ) return S_OK()
class Job: ############################################################################# def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ): """Instantiates the Workflow object and some default parameters. """ self.log = gLogger self.section = COMPONENT_NAME self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True #gConfig.getValue('Tier0SE-tape','SEName') self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo( disableVOMS = True ) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup( ret['Value']['group'] ) self.group = vo self.site = 'ANY' #ANY #self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] self.systemConfig = 'ANY' self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE', 'MinCPUTime': 'other.NAME<=VALUE', 'Site': 'other.NAME=="VALUE"', 'Platform': 'other.NAME=="VALUE"', #'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately 'BannedSites': 'other.Site!="VALUE"', 'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'} ##Add member to handle Parametric jobs self.parametric = {} self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow( script ) #Global error dictionary self.errorDict = {} ############################################################################# def setExecutable( self, executable, arguments = '', logFile = '' ): """Helper function. Specify executable script to run with optional arguments and log file for standard output. These can be either: - Submission of a python or shell script to DIRAC - Can be inline scripts e.g. C{'/bin/ls'} - Scripts as executables e.g. python or shell script file Example usage: >>> job = Job() >>> job.setExecutable('myScript.py') @param executable: Executable @type executable: string @param arguments: Optional arguments to executable @type arguments: string @param logFile: Optional log file name @type logFile: string """ kwargs = {'executable':executable, 'arguments':arguments, 'logFile':logFile} if not type( executable ) == type( ' ' ) or not type( arguments ) == type( ' ' ) or \ not type( logFile ) == type( ' ' ): return self._reportError( 'Expected strings for executable and arguments', **kwargs ) if os.path.exists( executable ): self.log.verbose( 'Found script executable file %s' % ( executable ) ) self.addToInputSandbox.append( executable ) logName = '%s.log' % ( os.path.basename( executable ) ) moduleName = os.path.basename( executable ) else: self.log.verbose( 'Found executable code' ) logName = 'CodeOutput.log' moduleName = 'CodeSegment' if logFile: if type( logFile ) == type( ' ' ): logName = logFile self.stepCount += 1 moduleName = moduleName.replace( '.', '' ) stepNumber = self.stepCount stepDefn = 'ScriptStep%s' % ( stepNumber ) step = self.__getScriptStep( stepDefn ) stepName = 'RunScriptStep%s' % ( stepNumber ) logPrefix = 'Script%s_' % ( stepNumber ) logName = '%s%s' % ( logPrefix, logName ) self.addToOutputSandbox.append( logName ) self.workflow.addStep( step ) # Define Step and its variables stepInstance = self.workflow.createStepInstance( stepDefn, stepName ) stepInstance.setValue( "name", moduleName ) stepInstance.setValue( "logFile", logName ) stepInstance.setValue( "executable", executable ) if arguments: stepInstance.setValue( "arguments", arguments ) return S_OK() ############################################################################# def setName( self, jobName ): """Helper function. A name for the job can be specified if desired. This will appear in the JobName field of the monitoring webpage. If nothing is specified a default value will appear. Example usage: >>> job=Job() >>> job.setName("myJobName") @param jobName: Name of job @type jobName: string """ kwargs = {'jobname':jobName} if not type( jobName ) == type( ' ' ): return self._reportError( 'Expected strings for job name', **kwargs ) else: self.workflow.setName( jobName ) self._addParameter( self.workflow, 'JobName', 'JDL', jobName, 'User specified name' ) return S_OK() ############################################################################# def setInputSandbox( self, files ): """Helper function. Specify input sandbox files less than 10MB in size. If over 10MB, files or a directory may be uploaded to Grid storage, see C{dirac.uploadSandbox()}. Paths to the options file and (if required) 'lib/' directory of the DLLs are specified here. Default is local directory. CMT requirements files or executables may be placed in the lib/ directory if desired. The lib/ directory is transferred to the Grid Worker Node before the job executes. Files / directories can be specified using the '*' character e.g. *.txt these are resolved correctly before job execution on the WN. Example usage: >>> job = Job() >>> job.setInputSandbox(['DaVinci.opts']) @param files: Input sandbox files, can specify full path @type files: Single string or list of strings ['',''] """ if type( files ) == list and len( files ): resolvedFiles = self._resolveInputSandbox( files ) fileList = ';'.join( resolvedFiles ) description = 'Input sandbox file list' self._addParameter( self.workflow, 'InputSandbox', 'JDL', fileList, description ) #self.sandboxFiles=resolvedFiles elif type( files ) == type( " " ): resolvedFiles = self._resolveInputSandbox( [files] ) fileList = ';'.join( resolvedFiles ) description = 'Input sandbox file' #self.sandboxFiles = [files] self._addParameter( self.workflow, 'InputSandbox', 'JDL', fileList, description ) else: kwargs = {'files':files} return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs ) return S_OK() ############################################################################# def setParametricInputSandbox( self, files ): """Helper function. Specify input sandbox files to used as parameters in the Parametric jobs. The possibilities are identical to the setInputSandbox. Example usage: >>> job = Job() >>> job.setParametricInputSandbox(['LFN:/Some_file','LFN:/Some_other_file']) @param files: Logical File Names @type files: Single LFN string or list of LFNs """ kwargs = {'files':files} if type( files ) == list and len( files ): for fileName in files: if not fileName.lower().count( "lfn:" ): return self._reportError( 'All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( files ) fileList = ";".join( resolvedFiles ) self.parametric['InputSandbox'] = fileList #self.sandboxFiles=resolvedFiles elif type( files ) == type( " " ): if not files.lower().count( "lfn:" ): return self._reportError( 'All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( [files] ) fileList = ";".join( resolvedFiles ) self.parametric['InputSandbox'] = fileList #self.sandboxFiles = [files] else: return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs ) return S_OK() ############################################################################# def setOutputSandbox( self, files ): """Helper function. Specify output sandbox files. If specified files are over 10MB, these may be uploaded to Grid storage with a notification returned in the output sandbox. Example usage: >>> job = Job() >>> job.setOutputSandbox(['DaVinci_v19r12.log','DVNTuples.root']) @param files: Output sandbox files @type files: Single string or list of strings ['',''] """ if type( files ) == list and len( files ): fileList = ';'.join( files ) description = 'Output sandbox file list' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description ) elif type( files ) == type( " " ): description = 'Output sandbox file' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description ) else: kwargs = {'files':files} return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs ) return S_OK() ############################################################################# def setInputData( self, lfns ): """Helper function. Specify input data by Logical File Name (LFN). Example usage: >>> job = Job() >>> job.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) @param lfns: Logical File Names @type lfns: Single LFN string or list of LFNs """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): lfns[i] = lfns[i].replace( 'LFN:', '' ) inputData = map( lambda x: 'LFN:' + x, lfns ) inputDataStr = ';'.join( inputData ) description = 'List of input data specified by LFNs' self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description ) elif type( lfns ) == type( ' ' ): #single LFN description = 'Input data specified by LFN' self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description ) else: kwargs = {'lfns':lfns} return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs ) return S_OK() ############################################################################# def setParametricInputData( self, lfns ): """Helper function. Specify input data by Logical File Name (LFN) to be used as a parameter in a parametric job Example usage: >>> job = Job() >>> job.setParametricInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) @param lfns: Logical File Names @type lfns: Single LFN string or list of LFNs """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): lfns[i] = lfns[i].replace( 'LFN:', '' ) inputData = map( lambda x: 'LFN:' + x, lfns ) inputDataStr = ';'.join( inputData ) self.parametric['InputData'] = inputDataStr elif type( lfns ) == type( ' ' ): #single LFN self.parametric['InputData'] = lfns else: kwargs = {'lfns':lfns} return self._reportError( 'Expected lfn string or list of lfns for parametric input data', **kwargs ) return S_OK() ############################################################################# def setInputDataPolicy( self, policy, dataScheduling = True ): """Helper function. Specify a job input data policy, this takes precedence over any site specific or global settings. Possible values for policy are 'Download' or 'Protocol' (case-insensitive). This requires that the module locations are defined for the VO in the CS. Example usage: >>> job = Job() >>> job.setInputDataPolicy('download') """ kwargs = {'policy':policy, 'dataScheduling':dataScheduling} csSection = '/Operations/InputDataPolicy' possible = ['Download', 'Protocol'] finalPolicy = '' for value in possible: if policy.lower() == value.lower(): finalPolicy = value if not finalPolicy: return self._reportError( 'Expected one of %s for input data policy' % ( ', '.join( possible ) ), __name__, **kwargs ) jobPolicy = gConfig.getValue( '%s/%s' % ( csSection, finalPolicy ), '' ) if not jobPolicy: return self._reportError( 'Could not get value for CS option %s/%s' % ( csSection, finalPolicy ), __name__, **kwargs ) description = 'User specified input data policy' self._addParameter( self.workflow, 'InputDataPolicy', 'JDL', jobPolicy, description ) if not dataScheduling and policy.lower() == 'download': self.log.verbose( 'Scheduling by input data is disabled, jobs will run anywhere and download input data' ) self._addParameter( self.workflow, 'DisableDataScheduling', 'JDL', 'True', 'Disable scheduling by input data' ) if not dataScheduling and policy.lower() != 'download': self.log.error( 'Expected policy to be "download" for bypassing data scheduling' ) return self._reportError( 'Expected policy to be "download" for bypassing data scheduling', __name__, **kwargs ) return S_OK() ############################################################################# def setOutputData( self, lfns, outputSE = None, outputPath = '' ): """Helper function. For specifying output data to be registered in Grid storage. If a list of OutputSEs are specified the job wrapper will try each in turn until successful. If the OutputPath is specified this will appear only after / <VO> / user / <initial> / <username> directory. Example usage: >>> job = Job() >>> job.setOutputData(['DVNtuple.root']) @param lfns: Output data file or files @type lfns: Single string or list of strings ['',''] @param outputSE: Optional parameter to specify the Storage Element @param outputPath: Optional parameter to specify part of the path in the storage (see above) Element to store data or files, e.g. CERN-tape @type outputSE: string or list @type outputPath: string """ if outputSE == None: outputSE = [] kwargs = {'lfns':lfns, 'OutputSE':outputSE, 'OutputPath':outputPath} if type( lfns ) == list and len( lfns ): outputDataStr = ';'.join( lfns ) description = 'List of output data files' self._addParameter( self.workflow, 'OutputData', 'JDL', outputDataStr, description ) elif type( lfns ) == type( " " ): description = 'Output data file' self._addParameter( self.workflow, 'OutputData', 'JDL', lfns, description ) else: return self._reportError( 'Expected file name string or list of file names for output data', **kwargs ) if outputSE: description = 'User specified Output SE' if type( outputSE ) in types.StringTypes: outputSE = [outputSE] elif type( outputSE ) != types.ListType: return self._reportError( 'Expected string or list for OutputSE', **kwargs ) outputSE = ';'.join( outputSE ) self._addParameter( self.workflow, 'OutputSE', 'JDL', outputSE, description ) if outputPath: description = 'User specified Output Path' if not type( outputPath ) in types.StringTypes: return self._reportError( 'Expected string for OutputPath', **kwargs ) # Remove leading "/" that might cause problems with os.path.join # FIXME: this will prevent to set OutputPath outside the Home of the User while outputPath[0] == '/': outputPath = outputPath[1:] self._addParameter( self.workflow, 'OutputPath', 'JDL', outputPath, description ) return S_OK() ############################################################################# def setPlatform( self, backend ): """Developer function. Choose submission pool on which job is executed e.g. DIRAC, LCG. Default in place for users. """ #should add protection here for list of supported platforms kwargs = {'backend':backend} if not type( backend ) == type( " " ): return self._reportError( 'Expected string for platform', **kwargs ) if not backend.lower() == 'any': self._addParameter( self.workflow, 'SubmitPools', 'JDL', backend, 'Platform type' ) return S_OK() ############################################################################# def setSystemConfig( self, config ): """Helper function. Choose system configuration (e.g. where user DLLs have been compiled). Default ANY in place for user jobs. Available system configurations can be browsed via dirac.checkSupportedPlatforms() method. Example usage: >>> job=Job() >>> job.setSystemConfig("slc4_ia32_gcc34") @param config: architecture, CMTCONFIG value @type config: string """ kwargs = {'config':config} if not type( config ) == type( " " ): return self._reportError( 'Expected string for system configuration', **kwargs ) description = 'User specified system configuration for job' self._addParameter( self.workflow, 'SystemConfig', 'JDLReqt', config, description ) self.systemConfig = config return S_OK() ############################################################################# def setCPUTime( self, timeInSecs ): """Helper function. Under Development. Specify CPU time requirement in DIRAC units. Example usage: >>> job = Job() >>> job.setCPUTime(5000) @param timeInSecs: CPU time @type timeInSecs: Int """ kwargs = {'timeInSecs':timeInSecs} if not type( timeInSecs ) == int: try: timeInSecs = int( timeInSecs ) except Exception: if not re.search( '{{', timeInSecs ): return self._reportError( 'Expected numerical string or int for CPU time in seconds', **kwargs ) description = 'CPU time in secs' self._addParameter( self.workflow, 'MaxCPUTime', 'JDLReqt', timeInSecs, description ) return S_OK() ############################################################################# def setDestination( self, destination ): """Helper function. Can specify a desired destination site or sites for job. This can be useful for debugging purposes but often limits the possible candidate sites and overall system response time. Example usage: >>> job = Job() >>> job.setDestination('LCG.CERN.ch') @param destination: site string @type destination: string or list """ kwargs = {'destination':destination} if type( destination ) == type( " " ): if not re.search( '^DIRAC.', destination ) and not destination.lower() == 'any': result = self.__checkSiteIsValid( destination ) if not result['OK']: return self._reportError( '%s is not a valid destination site' % ( destination ), **kwargs ) description = 'User specified destination site' self._addParameter( self.workflow, 'Site', 'JDLReqt', destination, description ) elif type( destination ) == list: for site in destination: if not re.search( '^DIRAC.', site ) and not site.lower() == 'any': result = self.__checkSiteIsValid( site ) if not result['OK']: return self._reportError( '%s is not a valid destination site' % ( destination ), **kwargs ) destSites = ';'.join( destination ) description = 'List of sites selected by user' self._addParameter( self.workflow, 'Site', 'JDLReqt', destSites, description ) else: return self._reportError( '%s is not a valid destination site, expected string' % ( destination ), **kwargs ) return S_OK() ############################################################################# def __checkSiteIsValid( self, site ): """Internal function to check that a site name is valid. """ sites = getSiteCEMapping() if not sites['OK']: return S_ERROR( 'Could not get site CE mapping' ) siteList = sites['Value'].keys() if not site in siteList: return S_ERROR( 'Specified site %s is not in list of defined sites' % site ) return S_OK( '%s is valid' % site ) ############################################################################# def setDestinationCE( self, ceName ): """ Developer function. Allows to direct a job to a particular Grid CE. """ kwargs = {'ceName':ceName} diracSite = getSiteForCE( ceName ) if not diracSite['OK']: return self._reportError( diracSite['Message'], **kwargs ) if not diracSite['Value']: return self._reportError( 'No DIRAC site name found for CE %s' % ( ceName ), **kwargs ) diracSite = diracSite['Value'] self.setDestination( diracSite ) self._addJDLParameter( 'GridRequiredCEs', ceName ) return S_OK() ############################################################################# def setBannedSites( self, sites ): """Helper function. Can specify a desired destination site for job. This can be useful for debugging purposes but often limits the possible candidate sites and overall system response time. Example usage: >>> job = Job() >>> job.setBannedSites(['LCG.GRIDKA.de','LCG.CNAF.it']) @param sites: single site string or list @type sites: string or list """ if type( sites ) == list and len( sites ): bannedSites = ';'.join( sites ) description = 'List of sites excluded by user' self._addParameter( self.workflow, 'BannedSites', 'JDLReqt', bannedSites, description ) elif type( sites ) == type( " " ): description = 'Site excluded by user' self._addParameter( self.workflow, 'BannedSites', 'JDLReqt', sites, description ) else: kwargs = {'sites':sites} return self._reportError( 'Expected site string or list of sites', **kwargs ) return S_OK() ############################################################################# def setOwner( self, ownerProvided ): """Developer function. Normally users should always specify their immutable DIRAC nickname. """ if not type( ownerProvided ) == type( " " ): return self._reportError( 'Expected string for owner', **{'ownerProvided':ownerProvided} ) self._addParameter( self.workflow, 'Owner', 'JDL', ownerProvided, 'User specified ID' ) return S_OK() ############################################################################# def setOwnerGroup( self, ownerGroup ): """Developer function. Allows to force expected owner group of proxy. """ if not type( ownerGroup ) == type( " " ): return self._reportError( 'Expected string for job owner group', **{'ownerGroup':ownerGroup} ) self._addParameter( self.workflow, 'OwnerGroup', 'JDL', ownerGroup, 'User specified owner group.' ) return S_OK() ############################################################################# def setType( self, jobType ): """Developer function. Specify job type for testing purposes. """ if not type( jobType ) == type( " " ): return self._reportError( 'Expected string for job type', **{'jobType':jobType} ) self._addParameter( self.workflow, 'JobType', 'JDL', jobType, 'User specified type' ) self.type = jobType return S_OK() ############################################################################# def _setSoftwareTags( self, tags ): """Developer function. Choose any software tags if desired. These are not compulsory but will ensure jobs only arrive at an LCG site where the software is preinstalled. Without the tags, missing software is installed automatically by the Job Agent. Example usage: >>> job=Job() >>> job.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10','VO-lhcb-Gauss-v25r12']) @param tags: software tags @type tags: string or list """ if type( tags ) == type( " " ): self._addParameter( self.workflow, 'SoftwareTag', 'JDL', tags, 'VO software tag' ) elif type( tags ) == list: swTags = ';'.join( tags ) self._addParameter( self.workflow, 'SoftwareTag', 'JDL', swTags, 'List of VO software tags' ) else: kwargs = {'tags':tags} return self._reportError( 'Expected String or List of software tags', **kwargs ) return S_OK() ############################################################################# def setJobGroup( self, jobGroup ): """Helper function. Allows to group certain jobs according to an ID. Example usage: >>> job = Job() >>> job.setJobGroup('Bs2JPsiPhi') @param jobGroup: JobGroup name @type jobGroup: string """ if not type( jobGroup ) == type( " " ): return self._reportError( 'Expected string for job group name', **{'jobGroup':jobGroup} ) description = 'User specified job group' self._addParameter( self.workflow, 'JobGroup', 'JDL', jobGroup, description ) return S_OK() ############################################################################# def setLogLevel( self, logLevel ): """Helper function. Optionally specify a DIRAC logging level for the job, e.g. ALWAYS, INFO, VERBOSE, WARN, DEBUG by default this is set to the info level. Example usage: >>> job = Job() >>> job.setLogLevel('debug') @param logLevel: Logging level @type logLevel: string """ kwargs = {'logLevel':logLevel} if type( logLevel ) in types.StringTypes: if logLevel.upper() in gLogger._logLevels.getLevels(): description = 'User specified logging level' self.logLevel = logLevel self._addParameter( self.workflow, 'LogLevel', 'JDL', logLevel, description ) else: return self._reportError( 'Error Level "%s" not valid' % logLevel, **kwargs ) else: return self._reportError( 'Expected string for logging level', **kwargs ) return S_OK() ############################################################################# def setConfigArgs( self, cfgString ): """Developer function. Allow to pass arbitrary settings to the payload configuration service environment. """ if not type( cfgString ) == type( " " ): return self._reportError( 'Expected string for DIRAC Job Config Args', **{'cfgString':cfgString} ) description = 'User specified cfg settings' self._addParameter( self.workflow, 'JobConfigArgs', 'JDL', cfgString, description ) return S_OK() ############################################################################# def setMode( self, mode ): """Developer function. Under development. """ if not type( mode ) == type( " " ): return self._reportError( 'Expected string for job mode', **{'mode':mode} ) description = 'Choose a different DIRAC job mode' self._addParameter( self.workflow, 'JobMode', 'JDL', mode, description ) return S_OK() ############################################################################# def selectSetup( self, setup ): """Developer function. Under development. """ if not type( setup ) == type( " " ): return self._reportError( 'Expected string for DIRAC setup', **{'setup':setup} ) description = 'Choose a different DIRAC setup in which to execute the job' self._addParameter( self.workflow, 'DIRACSetup', 'JDL', setup, description ) return S_OK() ############################################################################# def setExecutionEnv( self, environmentDict ): """Helper function. Optionally specify a dictionary of key, value pairs to be set before the job executes e.g. {'MYVAR':3} The standard application environment variables are always set so this is intended for user variables only. Example usage: >>> job = Job() >>> job.setExecutionEnv({'<MYVARIABLE>':'<VALUE>'}) @param environmentDict: Environment variables @type environmentDict: dictionary """ kwargs = {'environmentDict':environmentDict} if not type( environmentDict ) == type( {} ): return self._reportError( 'Expected dictionary of environment variables', **kwargs ) environment = [] for var, val in environmentDict.items(): try: environment.append( '='.join( [str( var ), str( val )] ) ) except Exception: return self._reportError( 'Expected string for environment variable key value pairs', **kwargs ) envStr = ';'.join( environment ) description = 'Env vars specified by user' self._addParameter( self.workflow, 'ExecutionEnvironment', 'JDL', envStr, description ) return S_OK() ############################################################################# def sendMail( self ): """Under development. """ description = 'Optional flag to send email when jobs complete' self._addParameter( self.workflow, 'SendMail', 'JDL', 'True', description ) ############################################################################# def createCode( self ): """Developer function. Wrapper method to create the code. """ return self.workflow.createCode() ############################################################################# def execute( self ): """Developer function. Executes the job locally. """ self.createCode() # code = self.createCode() # eval(compile(code,'<string>','exec')) self.workflow.execute() ############################################################################# def _getParameters( self ): """Developer function. Method to return the workflow parameters. """ wfParams = {} params = self.workflow.parameters for par in params: wfParams[par.getName()] = par.getValue() return wfParams ############################################################################# def _dumpParameters( self, showType = None ): """Developer function. Method to print the workflow parameters. """ paramsDict = {} paramList = self.workflow.parameters for param in paramList: paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()} self.log.info( '--------------------------------------' ) self.log.info( 'Workflow parameter summary: ' ) self.log.info( '--------------------------------------' ) #print self.workflow.parameters #print params.getParametersNames() for name, props in paramsDict.items(): ptype = paramsDict[name]['type'] value = paramsDict[name]['value'] if showType: if ptype == showType: self.log.info( 'NAME: %s\nTYPE: %s\nVALUE: %s ' % ( name, ptype, value ) ) self.log.info( '--------------------------------------' ) else: self.log.info( 'NAME: %s\nTYPE: %s\nVALUE: %s ' % ( name, ptype, value ) ) self.log.info( '--------------------------------------' ) ############################################################################# def __setJobDefaults( self ): """Set job default values. Note that the system configuration is set to "ANY". """ self._addParameter( self.workflow, 'JobType', 'JDL', self.type, 'Job Type' ) self._addParameter( self.workflow, 'Priority', 'JDL', self.priority, 'User Job Priority' ) self._addParameter( self.workflow, 'JobGroup', 'JDL', self.group, 'Name of the JobGroup' ) self._addParameter( self.workflow, 'JobName', 'JDL', self.name, 'Name of Job' ) #self._addParameter(self.workflow,'DIRACSetup','JDL',self.setup,'DIRAC Setup') self._addParameter( self.workflow, 'SystemConfig', 'JDLReqt', self.systemConfig, 'System configuration for job' ) self._addParameter( self.workflow, 'Site', 'JDL', self.site, 'Site Requirement' ) self._addParameter( self.workflow, 'Origin', 'JDL', self.origin, 'Origin of client' ) self._addParameter( self.workflow, 'StdOutput', 'JDL', self.stdout, 'Standard output file' ) self._addParameter( self.workflow, 'StdError', 'JDL', self.stderr, 'Standard error file' ) self._addParameter( self.workflow, 'InputData', 'JDL', '', 'Default null input data value' ) self._addParameter( self.workflow, 'LogLevel', 'JDL', self.logLevel, 'Job Logging Level' ) #Those 2 below are need for on-site resolution self._addParameter( self.workflow, 'ParametricInputData', 'JDL', '', 'Default null parametric input data value' ) self._addParameter( self.workflow, 'ParametricInputSandbox', 'JDL', '', 'Default null parametric input sandbox value' ) ############################################################################# def _addParameter( self, wObject, name, ptype, value, description, io = 'input' ): """ Internal Function Adds a parameter to the object. """ if io == 'input': inBool = True outBool = False elif io == 'output': inBool = False outBool = True else: raise TypeError, 'I/O flag is either input or output' par = Parameter( name, value, ptype, "", "", inBool, outBool, description ) wObject.addParameter( Parameter( parameter = par ) ) return par ############################################################################ def _resolveInputSandbox( self, inputSandbox ): """ Internal function. Resolves wildcards for input sandbox files. This is currently linux specific and should be modified. """ resolvedIS = [] for i in inputSandbox: if not re.search( '\*', i ): if not os.path.isdir( i ): resolvedIS.append( i ) for name in inputSandbox: if re.search( '\*', name ): #escape the star character... cmd = 'ls -d ' + name output = shellCall( 10, cmd ) if not output['OK']: self.log.error( 'Could not perform: %s' % ( cmd ) ) else: files = output['Value'].split() for check in files: if os.path.isfile( check ): self.log.verbose( 'Found file ' + check + ' appending to Input Sandbox' ) resolvedIS.append( check ) if os.path.isdir( check ): if re.search( '/$', check ): #users can specify e.g. /my/dir/lib/ check = check[:-1] tarname = os.path.basename( check ) directory = os.path.dirname( check ) #if just the directory this is null if directory: cmd = 'tar cfz ' + tarname + '.tar.gz ' + ' -C ' + directory + ' ' + tarname else: cmd = 'tar cfz ' + tarname + '.tar.gz ' + tarname output = shellCall( 60, cmd ) if not output['OK']: self.log.error( 'Could not perform: %s' % ( cmd ) ) resolvedIS.append( tarname + '.tar.gz' ) self.log.verbose( 'Found directory ' + check + ', appending ' + check + '.tar.gz to Input Sandbox' ) if os.path.isdir( name ): self.log.verbose( 'Found specified directory ' + name + ', appending ' + name + '.tar.gz to Input Sandbox' ) if re.search( '/$', name ): #users can specify e.g. /my/dir/lib/ name = name[:-1] tarname = os.path.basename( name ) directory = os.path.dirname( name ) #if just the directory this is null if directory: cmd = 'tar cfz ' + tarname + '.tar.gz ' + ' -C ' + directory + ' ' + tarname else: cmd = 'tar cfz ' + tarname + '.tar.gz ' + tarname output = shellCall( 60, cmd ) if not output['OK']: self.log.error( 'Could not perform: %s' % ( cmd ) ) else: resolvedIS.append( tarname + '.tar.gz' ) return resolvedIS ############################################################################# def __getScriptStep( self, name = 'Script' ): """Internal function. This method controls the definition for a script module. """ # Create the script module first moduleName = 'Script' module = ModuleDefinition( moduleName ) module.setDescription( 'A script module that can execute any provided script.' ) body = 'from DIRAC.Core.Workflow.Modules.Script import Script\n' module.setBody( body ) # Create Step definition step = StepDefinition( name ) step.addModule( module ) moduleInstance = step.createModuleInstance( 'Script', name ) # Define step parameters step.addParameter( Parameter( "name", "", "string", "", "", False, False, 'Name of executable' ) ) step.addParameter( Parameter( "executable", "", "string", "", "", False, False, 'Executable Script' ) ) step.addParameter( Parameter( "arguments", "", "string", "", "", False, False, 'Arguments for executable Script' ) ) step.addParameter( Parameter( "logFile", "", "string", "", "", False, False, 'Log file name' ) ) return step ############################################################################# def _toXML( self ): """Creates an XML representation of itself as a Job. """ return self.workflow.toXML() ############################################################################# def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir """Creates a JDL representation of itself as a Job. """ #Check if we have to do old bootstrap... classadJob = ClassAd( '[]' ) paramsDict = {} params = self.workflow.parameters # ParameterCollection object paramList = params for param in paramList: paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()} scriptname = 'jobDescription.xml' arguments = [] if self.script: if os.path.exists( self.script ): scriptname = os.path.abspath( self.script ) self.log.verbose( 'Found script name %s' % scriptname ) else: if xmlFile: self.log.verbose( 'Found XML File %s' % xmlFile ) scriptname = xmlFile arguments.append( os.path.basename( scriptname ) ) self.addToInputSandbox.append( scriptname ) if paramsDict.has_key( 'LogLevel' ): if paramsDict['LogLevel']['value']: arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) ) else: self.log.warn( 'Job LogLevel defined with null value' ) if paramsDict.has_key( 'DIRACSetup' ): if paramsDict['DIRACSetup']['value']: arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) ) else: self.log.warn( 'Job DIRACSetup defined with null value' ) if paramsDict.has_key( 'JobMode' ): if paramsDict['JobMode']['value']: arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) ) else: self.log.warn( 'Job Mode defined with null value' ) if paramsDict.has_key( 'JobConfigArgs' ): if paramsDict['JobConfigArgs']['value']: arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) ) else: self.log.warn( 'JobConfigArgs defined with null value' ) classadJob.insertAttributeString( 'Executable', self.executable ) self.addToOutputSandbox.append( self.stderr ) self.addToOutputSandbox.append( self.stdout ) #Extract i/o sandbox parameters from steps and any input data parameters #to do when introducing step-level api... #To add any additional files to input and output sandboxes if self.addToInputSandbox: extraFiles = ';'.join( self.addToInputSandbox ) if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] finalInputSandbox = currentFiles + ';' + extraFiles uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) ) paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox ) self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) ) else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = extraFiles paramsDict['InputSandbox']['type'] = 'JDL' if self.addToOutputSandbox: extraFiles = ';'.join( self.addToOutputSandbox ) if paramsDict.has_key( 'OutputSandbox' ): currentFiles = paramsDict['OutputSandbox']['value'] finalOutputSandbox = currentFiles + ';' + extraFiles uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) ) paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox ) self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) ) else: paramsDict['OutputSandbox'] = {} paramsDict['OutputSandbox']['value'] = extraFiles paramsDict['OutputSandbox']['type'] = 'JDL' if self.addToInputData: extraFiles = ';'.join( self.addToInputData ) if paramsDict.has_key( 'InputData' ): currentFiles = paramsDict['InputData']['value'] finalInputData = extraFiles if currentFiles: finalInputData = currentFiles + ';' + extraFiles uniqueInputData = uniqueElements( finalInputData.split( ';' ) ) paramsDict['InputData']['value'] = ';'.join( uniqueInputData ) self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) ) else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = extraFiles paramsDict['InputData']['type'] = 'JDL' ###Here handle the Parametric values if self.parametric: if self.parametric.has_key( 'InputData' ): if paramsDict.has_key( 'InputData' ): if paramsDict['InputData']['value']: currentFiles = paramsDict['InputData']['value'] + ";%s" paramsDict['InputData']['value'] = currentFiles else: paramsDict['InputData'] = {} paramsDict['InputData']['value'] = "%s" paramsDict['InputData']['type'] = 'JDL' self.parametric['files'] = self.parametric['InputData'] arguments.append( ' -p ParametricInputData=%s' ) elif self.parametric.has_key( 'InputSandbox' ): if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] + ";%s" paramsDict['InputSandbox']['value'] = currentFiles else: paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = '%s' paramsDict['InputSandbox']['type'] = 'JDL' self.parametric['files'] = self.parametric['InputSandbox'] arguments.append( ' -p ParametricInputSandbox=%s' ) if self.parametric.has_key( 'files' ): paramsDict['Parameters'] = {} paramsDict['Parameters']['value'] = self.parametric['files'] paramsDict['Parameters']['type'] = 'JDL' ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) ) #Add any JDL parameters to classad obeying lists with ';' rule requirements = False for name, props in paramsDict.items(): ptype = props['type'] value = props['value'] if name.lower() == 'requirements' and ptype == 'JDL': self.log.verbose( 'Found existing requirements: %s' % ( value ) ) requirements = True if re.search( '^JDL', ptype ): if not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... classadJob.insertAttributeString( name, value ) else: classadJob.insertAttributeVectorString( name, value.split( ';' ) ) if not requirements: reqtsDict = self.reqParams exprn = '' plus = '' for name, props in paramsDict.items(): ptype = paramsDict[name]['type'] value = paramsDict[name]['value'] if not ptype == 'dict': if ptype == 'JDLReqt': if value and not value.lower() == 'any': plus = ' && ' if re.search( ';', value ): for val in value.split( ';' ): exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus else: exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus if len( plus ): exprn = exprn[:-len( plus )] if not exprn: exprn = 'true' self.log.verbose( 'Requirements: %s' % ( exprn ) ) #classadJob.set_expression('Requirements', exprn) self.addToInputSandbox.remove( scriptname ) self.addToOutputSandbox.remove( self.stdout ) self.addToOutputSandbox.remove( self.stderr ) jdl = classadJob.asJDL() start = jdl.find( '[' ) end = jdl.rfind( ']' ) return jdl[( start + 1 ):( end - 1 )] ############################################################################# def _setParamValue( self, name, value ): """Internal Function. Sets a parameter value, used for production. """ return self.workflow.setValue( name, value ) ############################################################################# def _addJDLParameter( self, name, value ): """Developer function, add an arbitrary JDL parameter. """ self._addParameter( self.workflow, name, 'JDL', value, 'Optional JDL parameter added' ) return self.workflow.setValue( name, value ) ############################################################################# def _getErrors( self ): """Returns the dictionary of stored errors that will prevent submission or execution. """ return self.errorDict ############################################################################# def _reportError( self, message, name = '', **kwargs ): """Internal Function. Gets caller method name and arguments, formats the information and adds an error to the global error dictionary to be returned to the user. """ className = name if not name: className = __name__ methodName = sys._getframe( 1 ).f_code.co_name arguments = [] for key in kwargs: if kwargs[key]: arguments.append( '%s = %s ( %s )' % ( key, kwargs[key], type( kwargs[key] ) ) ) finalReport = 'Problem with %s.%s() call:\nArguments: %s\nMessage: %s\n' % ( className, methodName, ', '.join( arguments ), message ) if self.errorDict.has_key( methodName ): tmp = self.errorDict[methodName] tmp.append( finalReport ) self.errorDict[methodName] = tmp else: self.errorDict[methodName] = [finalReport] self.log.verbose( finalReport ) return S_ERROR( finalReport )
print( 'Test application execution' ) print 'Module MODULE.getDescrShort() =', self.MODULE.getDescrShort() print 'Module MODULE_DEFINITION_NAME =', self.MODULE_DEFINITION_NAME print 'Module MODULE_INSTANCE_NAME =', self.MODULE_INSTANCE_NAME print 'Module STEP.getDescrShort() =', self.STEP.getDescrShort() print 'Module STEP_DEFINITION_NAME =', self.STEP_DEFINITION_NAME print 'Module STEP_INSTANCE_NAME =', self.STEP_INSTANCE_NAME return 1""" md1 = ModuleDefinition("TestAppModule") md1.addParameter( Parameter("enable", "True", "bool", "", "", True, False, "If False execution is disabled")) md1.setBody(bodyTestApp) md1.setDescrShort("Empty Module") md1.setDescription("Empty Module to do some testing") md1.setRequired("") md1.setOrigin("") md1.setVersion(0.1) sd1 = StepDefinition("TestAppStep") sd1.addModule(md1) mi1 = sd1.createModuleInstance("TestAppModule", "testappmod1") wf1 = Workflow("TestAppWF") wf1.addStep(sd1) si1 = wf1.createStepInstance("TestAppStep", "testappstep1") print(wf1.createCode()) # eval(compile(wf1.createCode(),'<string>','exec')) wf1.execute()
print 'Module ',str(type(self)),'is disabled' return 1 print( 'Test application execution' ) print 'Module MODULE.getDescrShort() =', self.MODULE.getDescrShort() print 'Module MODULE_DEFINITION_NAME =', self.MODULE_DEFINITION_NAME print 'Module MODULE_INSTANCE_NAME =', self.MODULE_INSTANCE_NAME print 'Module STEP.getDescrShort() =', self.STEP.getDescrShort() print 'Module STEP_DEFINITION_NAME =', self.STEP_DEFINITION_NAME print 'Module STEP_INSTANCE_NAME =', self.STEP_INSTANCE_NAME return 1""" md1 = ModuleDefinition("TestAppModule") md1.addParameter(Parameter("enable", "True", "bool", "", "", True, False, "If False execution is disabled")) md1.setBody(bodyTestApp) md1.setDescrShort("Empty Module") md1.setDescription("Empty Module to do some testing") md1.setRequired("") md1.setOrigin("") md1.setVersion(0.1) sd1 = StepDefinition("TestAppStep") sd1.addModule(md1) mi1 = sd1.createModuleInstance("TestAppModule", "testappmod1") wf1 = Workflow("TestAppWF") wf1.addStep(sd1) si1 = wf1.createStepInstance("TestAppStep", "testappstep1") print wf1.createCode() # eval(compile(wf1.createCode(),'<string>','exec')) wf1.execute()
mtuple = (1, 2, 3, 4, 5) mi9.findParameter('message').setValue(mtuple, 'tuple') mstring = """\"Clever string of mine; WR.Output = \"Collection=\'EVTTAGS/TagCreator/1\' ADDRESS=\'/Event\' DATAFILE=\"""" #mstring = "Clever string of mine;" mi10.findParameter('message').setValue(mstring, 'string') mbool = False mi11.findParameter('message').setValue(mbool, 'bool') mint = 12672 mi12.findParameter('message').setValue(mint, 'int') mlistdict = [{'SORTIE_@{inparam4}': 4098, 'sape_@{inpar2}': 4139},{"@{inparam4}jj@{inpar2}":234}] mi12.findParameter('message').setValue(mlistdict, 'list') sd1.findParameter('result').link('mi5','result') #sd1.findParameter('message').link('self','inparam4') # taken from the level of step w1 = Workflow(name='main') w1.setOrigin('/home/user/blablabla') w1.setDescription("Pretty long description\n several lines of text") w1.setDescrShort("Oooooo short description") w1.addStep(sd1) w1.addParameter(Parameter("final","0.0","float","","",False, True, "Final result")) w1.addParameter(Parameter("debug","False","bool","","",True, False, "Debug switch")) w1.addParameter(Parameter("message","vv@{inparam4}jj@{inpar2}ge","string","","",True, False, "")) w1.addParameter(Parameter("inparam4","VER","string","","",True, False, "")) w1.addParameter(Parameter("inpar2","SORTIE@{inparam4}","string","","",True, False, "")) si1 = w1.createStepInstance('TotalSumm', 'si1') si2 = w1.createStepInstance('TotalSumm', 'si2') si1.findParameter('debug').link('self','debug') si2.findParameter('debug').link('self','debug')
mbool = False mi11.findParameter('message').setValue(mbool, 'bool') mint = 12672 mi12.findParameter('message').setValue(mint, 'int') mlistdict = [{ 'SORTIE_@{inparam4}': 4098, 'sape_@{inpar2}': 4139 }, { "@{inparam4}jj@{inpar2}": 234 }] mi12.findParameter('message').setValue(mlistdict, 'list') sd1.findParameter('result').link('mi5', 'result') #sd1.findParameter('message').link('self','inparam4') # taken from the level of step w1 = Workflow(name='main') w1.setOrigin('/home/user/blablabla') w1.setDescription("Pretty long description\n several lines of text") w1.setDescrShort("Oooooo short description") w1.addStep(sd1) w1.addParameter( Parameter("final", "0.0", "float", "", "", False, True, "Final result")) w1.addParameter( Parameter("debug", "False", "bool", "", "", True, False, "Debug switch")) w1.addParameter( Parameter("message", "vv@{inparam4}jj@{inpar2}ge", "string", "", "", True, False, "")) w1.addParameter(Parameter("inparam4", "VER", "string", "", "", True, False, "")) w1.addParameter(
if not self.enable: print 'Module ',str(type(self)),'is disabled' return 1 print( 'Test application execution' ) print 'Module MODULE.getDescrShort() =', self.MODULE.getDescrShort() print 'Module MODULE_DEFINITION_NAME =', self.MODULE_DEFINITION_NAME print 'Module MODULE_INSTANCE_NAME =', self.MODULE_INSTANCE_NAME print 'Module STEP.getDescrShort() =', self.STEP.getDescrShort() print 'Module STEP_DEFINITION_NAME =', self.STEP_DEFINITION_NAME print 'Module STEP_INSTANCE_NAME =', self.STEP_INSTANCE_NAME return 1""" md1 = ModuleDefinition('TestAppModule') md1.appendParameter( Parameter("enable","True","bool","","",True,False,"If False execution is disabled")) md1.setBody(bodyTestApp) md1.setDescrShort('Empty Module') md1.setDescription('Empty Module to do some testing') md1.setRequired('') md1.setOrigin('') md1.setVersion(0.1) sd1 = StepDefinition('TestAppStep') sd1.addModule(md1) mi1 = sd1.createModuleInstance('TestAppModule', 'testappmod1') wf1 = Workflow('TestAppWF') wf1.addStep(sd1) si1 = wf1.createStepInstance('TestAppStep','testappstep1') print wf1.createCode() #eval(compile(wf1.createCode(),'<string>','exec')) wf1.execute()