def createSingleModuleWorkflow(module, name): ''' Creates a workflow based on a single module definition ''' moduleType = module.getType() moduleName = name workflow = Workflow() step = StepDefinition(moduleType + '_step') step.addModule(module) moduleInstance = step.createModuleInstance(moduleType, moduleName) step.addParameter(moduleInstance.parameters.getInput()) workflow.addParameter(moduleInstance.parameters.getInput()) workflow.addStep(step) stepInstance = workflow.createStepInstance(moduleType + '_step', moduleName + '_step') # Propagate the module input parameters to the workflow level moduleInstance.linkParameterUp(moduleInstance.parameters.getInput()) stepInstance.linkParameterUp(moduleInstance.parameters.getInput()) workflow.setName(name) workflow.setDescription('Single module workflow from ' + moduleType + ' type module') workflow.setDescrShort(moduleType + ' workflow') return workflow
def __init__(self, script, parameters=None, softwarePackage='HESS/v0.1/root', compiled=False, cpuTime=3600): Job.__init__(self) self.workflow = Workflow() self.executable = '$DIRACROOT/scripts/cta-root-macro' self.setName(os.path.basename(script)) self.script = os.path.basename(script) self.setInputSandbox([script]) self.setCPUTime(cpuTime) arguments = [] toCompile = '' if compiled: toCompile = '+' if parameters: arguments = [ repr(k).replace('"', "\\\\'").replace("'", "\\\\'") for k in parameters ] argumentStr = "%s%s %s" % (os.path.basename(script), toCompile, ' '.join(arguments)) self.setConfigArgs(argumentStr) self.__addSoftwarePackage(softwarePackage)
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ): """Instantiates the Workflow object and some default parameters. """ super( Job, self ).__init__() self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True #gConfig.getValue('Tier0SE-tape','SEName') self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo( disableVOMS = True ) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup( ret['Value']['group'] ) self.group = vo self.site = 'ANY' #ANY #self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] self.systemConfig = 'ANY' self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE', 'MinCPUTime': 'other.NAME<=VALUE', 'Site': 'other.NAME=="VALUE"', 'Platform': 'other.NAME=="VALUE"', #'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately 'BannedSites': 'other.Site!="VALUE"', 'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'} ##Add member to handle Parametric jobs self.parametric = {} self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow( script )
def __init__(self, cpuTime=3600): Job.__init__(self) self.workflow = Workflow() self.executable = '$DIRACROOT/scripts/cta-corsikasimteluser' self.setCPUTime(cpuTime) global argumentStr
def __init__(self, script=None, stdout='std.out', stderr='std.err'): """Instantiates the Workflow object and some default parameters. """ super(Job, self).__init__() self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo(disableVOMS=True) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup(ret['Value']['group']) self.group = vo self.site = None # self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified # $DIRACROOT is set by the JobWrapper at execution time self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] # #Add member to handle Parametric jobs self.numberOfParameters = 0 self.parameterSeqs = {} self.wfArguments = {} self.parametricWFArguments = {} # loading the function that will be used to determine the platform (it can be VO specific) res = ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", 'getDIRACPlatforms') if not res['OK']: self.log.fatal(res['Message']) self.getDIRACPlatforms = res['Value'] self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow(script) self._siteSet = set(getSites().get('Value', []))
def __init__(self, parameters=None, cpuTime=3600): Job.__init__(self) self.workflow = Workflow() self.executable = '$DIRACROOT/scripts/cta-hap-wf-param' self.setCPUTime(cpuTime) argumentStr = "%s" % (' '.join(parameters)) self.setConfigArgs(argumentStr)
def __init__(self, cpuTime=3600): Job.__init__(self) self.workflow = Workflow() self.executable = '$DIRACROOT/scripts/cta-read-cta' # self.executable = '$DIRACROOT/CTADIRAC/Core/scripts/cta-read_cta.py' self.setCPUTime(cpuTime) global argumentStr
def __init__(self, cpuTime=3600): Job.__init__(self) self.workflow = Workflow() self.executable = '$DIRACROOT/scripts/cta-evndispstandalone' self.setCPUTime(cpuTime) global parfileList parfileList = [] global argumentStr
def __init__(self, script, parameters=None, cpuTime=3600): Job.__init__(self) self.workflow = Workflow() self.executable = '$DIRACROOT/scripts/cta-hap-application' self.setName(os.path.basename(script)) self.setCPUTime(cpuTime) global argumentStr argumentStr = "%s" % (' '.join(parameters)) self.setConfigArgs(argumentStr)
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ): """Instantiates the Workflow object and some default parameters. """ super( Job, self ).__init__() self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True #gConfig.getValue('Tier0SE-tape','SEName') self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo( disableVOMS = True ) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup( ret['Value']['group'] ) self.group = vo self.site = 'ANY' #ANY #self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] ##Add member to handle Parametric jobs self.parametric = {} self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow( script )
def startElement(self, name, attrs): # print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames() self.clearCharacters( ) # clear to remove empty or nonprintable characters if name == "Workflow": if self.root is None: # if root not defined by constractor self.root = Workflow() self.stack.append(self.root) elif name == "StepDefinition": obj = StepDefinition("TemporaryXMLObject_StepDefinition") if self.root is None: # in case we are saving Step only self.root = obj self.stack.append(obj) elif name == "StepInstance": obj = StepInstance("TemporaryXMLObject_StepInstance") self.stack.append(obj) elif name == "ModuleDefinition": obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition") if self.root is None: # in case we are saving Module only self.root = obj self.stack.append(obj) elif name == "ModuleInstance": obj = ModuleInstance("TemporaryXMLObject_ModuleInstance") self.stack.append(obj) elif name == "Parameter": obj = Parameter(str(attrs['name']), None, str(attrs['type']), str(attrs['linked_module']), str(attrs['linked_parameter']), str(attrs['in']), str(attrs['out']), str(attrs['description'])) self.stack.append(obj) # TEMPORARY CODE # 2020-03-27: this is temporary code since 2007-06-26 elif name == "origin" or name == "version" or name == "name"\ or name == "type" or name == "value"\ or name == "required" or name == "descr_short"\ or name == "name" or name == "type" or name == "description" or name == "body": pass else: print("UNTREATED! startElement name=", name, "attr=", attrs.getLength(), attrs.getNames()) pass
def __getProductionParameters(self, prodXMLFile, prodID, groupDescription='', bkPassInfo={}, derivedProd=0, reqID=0): """ This method will publish production parameters. """ prodWorkflow = Workflow(prodXMLFile) parameters = {} info = [] for parameterName in ('Priority', 'CondDBTag', 'DDDBTag', 'DQTag', 'eventType', 'processingPass', 'FractionToProcess', 'MinFilesToProcess', 'configName', 'configVersion', 'outputDataFileMask', 'JobType', 'MaxNumberOfTasks'): try: parameters[parameterName] = prodWorkflow.findParameter( parameterName).getValue() info.append( "%s: %s" % (parameterName, prodWorkflow.findParameter(parameterName).getValue())) except AttributeError: continue parameters['SizeGroup'] = self.jobFileGroupSize if prodWorkflow.findParameter( 'InputData'): # now only comes from BK query prodWorkflow.findParameter('InputData').setValue('') gLogger.verbose( 'Resetting input data for production to null, this comes from a BK query...' ) prodXMLFile = self.__createWorkflow(prodXMLFile) # prodWorkflow.toXMLFile(prodXMLFile) if self.transformationFamily: parameters['TransformationFamily'] = self.transformationFamily if not bkPassInfo: bkPassInfo = prodWorkflow.findParameter( 'BKProcessingPass').getValue() if not groupDescription: groupDescription = prodWorkflow.findParameter( 'groupDescription').getValue() parameters['BKCondition'] = prodWorkflow.findParameter( 'conditions').getValue() parameters['BKProcessingPass'] = bkPassInfo parameters['groupDescription'] = groupDescription parameters['RequestID'] = reqID parameters['DerivedProduction'] = derivedProd result = self.__getOutputLFNs(prodID, '99999999', prodXMLFile) if not result['OK']: gLogger.error('Could not create production LFNs', result) outputLFNs = result['Value'] parameters['OutputLFNs'] = outputLFNs # the list of output directories is later used for consistency check and for removing output data outputDirectories = [] del outputLFNs[ 'BookkeepingLFNs'] # since ProductionOutputData uses the file mask for i in outputLFNs.values(): for j in i: outputDir = '%s%s' % (j.split(str(prodID))[0], prodID) if outputDir not in outputDirectories: outputDirectories.append(outputDir) parameters['OutputDirectories'] = outputDirectories # Now for the steps of the workflow stepKeys = sorted(bkPassInfo.keys()) for step in stepKeys: info.append('====> %s %s %s' % (bkPassInfo[step]['ApplicationName'], bkPassInfo[step]['ApplicationVersion'], step)) info.append('%s Option Files:' % (bkPassInfo[step]['ApplicationName'])) if bkPassInfo[step]['OptionFiles']: for opts in bkPassInfo[step]['OptionFiles'].split(';'): info.append('%s' % opts) if bkPassInfo[step]['ExtraPackages']: info.append('ExtraPackages: %s' % (bkPassInfo[step]['ExtraPackages'])) try: if parameters['BkQuery']: info.append('\nBK Input Data Query:') for bkn, bkv in parameters['BkQuery'].iteritems(): info.append('%s= %s' % (bkn, bkv)) except KeyError: pass # BK output directories (very useful) bkPaths = [] bkOutputPath = '%s/%s/%s/%s/%s/%s' % ( parameters['configName'], parameters['configVersion'], parameters['BKCondition'], parameters.get('processingPass', ''), parameters['groupDescription'], parameters['eventType']) fileTypes = parameters['outputDataFileMask'] fileTypes = [a.upper() for a in fileTypes.split(';')] # Annoying that histograms are extension root if 'ROOT' in fileTypes: fileTypes.remove('ROOT') fileTypes.append('HIST') for ft in fileTypes: bkPaths.append('%s/%s' % (bkOutputPath, ft)) parameters['BKPaths'] = bkPaths info.append('\nBK Browsing Paths:\n%s' % ('\n'.join(bkPaths))) infoString = '\n'.join(info) parameters['DetailedInfo'] = infoString gLogger.verbose('Parameters that will be added: %s' % parameters) return parameters