def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if self.WorkflowStartFrom: self.startFrom = self.WorkflowStartFrom if 'stdhepFile' in self.step_commons: inputf = self.step_commons["stdhepFile"] if not isinstance( inputf, list ): inputf = inputf.split(";") self.InputFile = inputf self.RandomSeed = self.determineRandomSeed() resDBSlice = self.determineDBSlice() if not resDBSlice['OK']: return resDBSlice if "IS_PROD" in self.workflow_commons: if self.workflow_commons["IS_PROD"]: self.mcRunNumber = self.RandomSeed #self.OutputFile = getProdFilename(self.outputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_sim_"): self.OutputFile = os.path.basename(obj) elif obj.lower().count("_gen_"): self.InputFile = [os.path.basename(obj)] else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]), self.workflow_commons, ) #if 'WhizardOutput' in self.workflow_commons: # self.InputFile = getProdFilename(self.workflow_commons["WhizardOutput"], # int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]))] if len(self.InputData): if 'EvtClass' in self.inputdataMeta and not self.ProcessID: self.ProcessID = self.inputdataMeta['EvtClass'] if 'EvtType' in self.inputdataMeta and not self.ProcessID: self.ProcessID = self.inputdataMeta['EvtType'] if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".stdhep") > -1 or files.lower().find(".hepevt") > -1: self.InputFile.append(files) break return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if 'ParametricInputSandbox' in self.workflow_commons: paramsb = self.workflow_commons['ParametricInputSandbox'] if not isinstance( paramsb, list ): if paramsb: paramsb = paramsb.split(";") else: paramsb = [] self.InputFile += paramsb ##Backward compat needed, cannot remove yet. self.outputREC = self.step_commons.get('outputREC', self.outputREC) self.outputDST = self.step_commons.get('outputDST', self.outputDST) if 'IS_PROD' in self.workflow_commons: if self.workflow_commons["IS_PROD"] and not self.OutputFile: #self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #if 'MokkaOutput' in self.workflow_commons: # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_rec_"): self.outputREC = os.path.basename(obj) elif obj.lower().count("_dst_"): self.outputDST = os.path.basename(obj) elif obj.lower().count("_sim_"): self.InputFile = [os.path.basename(obj)] else: self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) #if 'MokkaOutput' in self.workflow_commons: # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]))] if not self.InputFile and self.InputData: for files in self.InputData: if files.lower().find(".slcio") > -1: self.InputFile.append(files) return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if self.workflow_commons.has_key('ParametricInputSandbox'): paramsb = self.workflow_commons['ParametricInputSandbox'] if not type(paramsb) == types.ListType: if len(paramsb): paramsb = paramsb.split(";") else: paramsb = [] self.InputFile += paramsb ##Backward compat needed, cannot remove yet. self.outputREC = self.step_commons.get('outputREC', self.outputREC) self.outputDST = self.step_commons.get('outputDST', self.outputDST) if self.workflow_commons.has_key("IS_PROD"): if self.workflow_commons["IS_PROD"] and len(self.OutputFile)==0: #self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #if self.workflow_commons.has_key("MokkaOutput"): # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if self.workflow_commons.has_key('ProductionOutputData'): outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_rec_"): self.outputREC = os.path.basename(obj) elif obj.lower().count("_dst_"): self.outputDST = os.path.basename(obj) elif obj.lower().count("_sim_"): self.InputFile = [os.path.basename(obj)] else: self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) #if self.workflow_commons.has_key("MokkaOutput"): # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]))] if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".slcio") > -1: self.InputFile.append(files) return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Called from ModuleBase """ if 'NbEvtsKept' in self.step_commons: self.NbEvtsKept = self.step_commons['NbEvtsKept'] if not self.NbEvtsKept: return S_ERROR('Nb of events to keep MUST be specified') if 'IS_PROD' in self.workflow_commons: if self.workflow_commons["IS_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons[ 'ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_gen_"): self.InputFile = [os.path.basename(obj)] self.OutputFile = self.InputFile[0] else: if 'WhizardOutput' in self.workflow_commons: self.stdhepFile = getProdFilename( self.workflow_commons["WhizardOutput"], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().count(".stdhep"): self.InputFile.append(files) break return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve LCIO concatenate specific parameters, called from ModuleBase """ if not self.OutputFile: return S_ERROR( 'No output file defined' ) if self.workflow_commons.has_key("IS_PROD"): if self.workflow_commons["IS_PROD"]: if self.workflow_commons.has_key('ProductionOutputData'): self.prod_outputdata = self.workflow_commons['ProductionOutputData'].split(";") for obj in self.prod_outputdata: if obj.lower().count("_gen_"): self.OutputFile = os.path.basename(obj) else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".stdhep") > -1: self.InputFile.append(files) if self.step_commons.has_key('listoutput'): if len(self.step_commons['listoutput']): self.listoutput = self.step_commons['listoutput'][0] return S_OK('Parameters resolved')
def applicationSpecificInputs(self): if self.step_commons.has_key('CutFile'): self.SteeringFile = self.step_commons['CutFile'] if self.step_commons.has_key('MaxNbEvts'): self.MaxNbEvts = self.step_commons['MaxNbEvts'] if not self.OutputFile: dircont = os.listdir("./") for myfile in dircont: if myfile.count(".stdhep"): self.OutputFile = myfile.rstrip(".stdhep") + "_reduced.stdhep" break if not self.OutputFile: return S_ERROR("Could not find suitable OutputFile name") if self.workflow_commons.has_key("IS_PROD"): if self.workflow_commons["IS_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if self.workflow_commons.has_key('ProductionOutputData'): outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_gen_"): self.OutputFile = os.path.basename(obj) break else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) return S_OK()
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if self.WorkflowStartFrom: self.startFrom = self.WorkflowStartFrom ##Move below to ModuleBase as common to Mokka if "IS_PROD" in self.workflow_commons: self.RandomSeed = int( str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"]))) elif self.jobID: self.RandomSeed = self.jobID if 'IS_PROD' in self.workflow_commons and self.workflow_commons[ "IS_PROD"]: self.OutputFile = getProdFilename( self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".stdhep") > -1 or files.lower().find( ".hepevt") > -1: self.InputFile.append(files) break return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Called from ModuleBase """ if 'NbEvtsKept' in self.step_commons: self.NbEvtsKept = self.step_commons['NbEvtsKept'] if not self.NbEvtsKept: return S_ERROR('Nb of events to keep MUST be specified') if 'IS_PROD' in self.workflow_commons: if self.workflow_commons["IS_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_gen_"): self.InputFile = [os.path.basename(obj)] self.OutputFile = self.InputFile[0] else: if 'WhizardOutput' in self.workflow_commons: self.stdhepFile = getProdFilename(self.workflow_commons["WhizardOutput"], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().count(".stdhep"): self.InputFile.append(files) break return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve LCIO concatenate specific parameters, called from ModuleBase """ if not self.OutputFile: return S_ERROR('No output file defined') if 'IS_PROD' in self.workflow_commons and self.workflow_commons[ "IS_PROD"]: if 'ProductionOutputData' in self.workflow_commons: self.prod_outputdata = self.workflow_commons[ 'ProductionOutputData'].split(";") for obj in self.prod_outputdata: if obj.lower().count("_gen_"): self.OutputFile = os.path.basename(obj) else: self.OutputFile = getProdFilename( self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".stdhep") > -1: self.InputFile.append(files) if 'listoutput' in self.step_commons: if len(self.step_commons['listoutput']): self.listoutput = self.step_commons['listoutput'][0] return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if self.WorkflowStartFrom: self.startFrom = self.WorkflowStartFrom ##Move below to ModuleBase as common to Mokka if "IS_PROD" in self.workflow_commons: self.RandomSeed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"]))) elif self.jobID: self.RandomSeed = self.jobID if 'IS_PROD' in self.workflow_commons and self.workflow_commons["IS_PROD"]: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".stdhep") > -1 or files.lower().find(".hepevt") > -1: self.InputFile.append(files) break return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if self.WorkflowStartFrom: self.startFrom = self.WorkflowStartFrom self.randomSeed = self._determineRandomSeed() if "IS_PROD" in self.workflow_commons and self.workflow_commons[ "IS_PROD"]: self.OutputFile = getProdFilename( self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]), self.workflow_commons, ) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.endswith(DDSIMINPUTFORMATS): self.InputFile.append(files) return S_OK('Parameters resolved')
def applicationSpecificInputs(self): if not self.OutputFile: dircont = os.listdir("./") for myfile in dircont: if myfile.count(".stdhep"): self.OutputFile = myfile.replace(".stdhep", "_reduced.stdhep") break if not self.OutputFile: return S_ERROR("Could not find suitable OutputFile name") if "IS_PROD" in self.workflow_commons and self.workflow_commons["IS_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_gen_"): self.OutputFile = os.path.basename(obj) break else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) self.log.notice("Outputfile: %s" % self.OutputFile) if self.inlineCuts: cfile = open("cuts_local.txt", "w") cfile.write("\n".join(self.inlineCuts.split(";"))) cfile.close() self.SteeringFile = "cuts_local.txt" return S_OK()
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if 'ParametricInputSandbox' in self.workflow_commons: paramsb = self.workflow_commons['ParametricInputSandbox'] if not isinstance( paramsb, list ): if paramsb: paramsb = paramsb.split(";") else: paramsb = [] self.InputFile += paramsb ##Backward compat needed, cannot remove yet. self.outputREC = self.step_commons.get('outputREC', self.outputREC) self.outputDST = self.step_commons.get('outputDST', self.outputDST) if 'IS_PROD' in self.workflow_commons: if self.workflow_commons["IS_PROD"] and not self.OutputFile: if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") LOG.info('Found ProductionOutputData in WF Commons', str(outputlist)) for obj in outputlist: if obj.lower().count("_rec_"): self.outputREC = os.path.basename(obj) elif obj.lower().count("_dst_"): self.outputDST = os.path.basename(obj) elif obj.lower().count("_sim_"): self.InputFile = [os.path.basename(obj)] else: prodID = int(self.workflow_commons['PRODUCTION_ID']) jobID = int(self.workflow_commons["JOB_ID"]) self.outputREC = getProdFilename(self.outputREC, prodID, jobID) self.outputDST = getProdFilename(self.outputDST, prodID, jobID) self.InputFile = [getProdFilename(self.InputFile, prodID, jobID)] LOG.info('OutputDST:', self.outputDST) LOG.info('OutputREC:', self.outputREC) if not self.InputFile and self.InputData: for files in self.InputData: if files.lower().find(".slcio") > -1: self.InputFile.append(files) return S_OK('Parameters resolved')
def constructProductionLFNs(paramDict): """ Used for local testing of a workflow, a temporary measure until LFN construction is tidied. This works using the workflow commons for on the fly construction. :param dict paramDict: dictionary with at least the keys ``PRODUCTION_ID``, ``JOB_ID``, ``outputList`` :returns: S_OK with ``ProductionOutputData``, ``LogFilePath``, ``LogTargetPath`` """ result = checkForMandatoryKeys(paramDict, ['PRODUCTION_ID', 'JOB_ID', 'outputList']) if not result['OK']: return result productionID = paramDict['PRODUCTION_ID'] jobID = paramDict['JOB_ID'] outputList = paramDict['outputList'] fileTupleList = [] gLogger.verbose('outputList %s' % (outputList)) for info in outputList: #Nasty check on whether the created code parameters were not updated e.g. when changing defaults in a workflow fileName = info['outputFile'] #rename to take care of correct path fileName = getProdFilename(fileName, int(productionID), int(jobID)) fileTupleList.append((info['outputPath'], fileName)) #Get all LFN(s) to output data outputData = [] for fileTuple in fileTupleList: lfn = fileTuple[0] + "/" + str(productionID).zfill(8) + "/" + str(int(jobID)/1000).zfill(3) + "/" + fileTuple[1] lfn = cleanUpLFNPath(lfn) outputData.append(lfn) #Get log file path - unique for all modules ## get First outputfile basePath = fileTupleList[0][0] #TODO adjust for ILD logPath = basePath + "/" + str(productionID).zfill(8) + "/LOG" #used for logFile upload to the LogSE logFilePath = [cleanUpLFNPath('%s/%s' % (logPath, str(int(jobID)/1000).zfill(3)))] resLogs = getLogPath( dict(PRODUCTION_ID=productionID, JOB_ID=jobID, LogFilePath=logFilePath), basePath=basePath ) if not resLogs['OK']: return resLogs logTargetPath = resLogs['Value']['LogTargetPath'] logFilePath = resLogs['Value']['LogFilePath'] if not outputData: gLogger.info('No output data LFN(s) constructed') else: gLogger.verbose('Created the following output data LFN(s):\n%s' % (string.join(outputData,'\n'))) gLogger.verbose('Log file path is:\n%s' % logFilePath[0]) gLogger.verbose('Log target path is:\n%s' % logTargetPath[0]) jobOutputs = {'ProductionOutputData' : outputData, 'LogFilePath' : logFilePath, 'LogTargetPath' : logTargetPath} return S_OK(jobOutputs)
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ self.randomSeed = self._determineRandomSeed() if "IS_PROD" in self.workflow_commons and self.workflow_commons["IS_PROD"]: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]), self.workflow_commons, ) return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ self.randomSeed = self._determineRandomSeed() if "IS_PROD" in self.workflow_commons and self.workflow_commons[ "IS_PROD"]: self.OutputFile = getProdFilename( self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]), self.workflow_commons, ) return S_OK('Parameters resolved')
def applicationSpecificInputs(self): if 'IS_PROD' in self.workflow_commons: if self.workflow_commons["IS_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") baseoutputfile = self.OutputFile.split(".stdhep")[0] for obj in outputlist: if obj.count(baseoutputfile): self.OutputFile = os.path.basename(obj) else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) return S_OK()
def applicationSpecificInputs(self): """ Resolve LCIO concatenate specific parameters, called from ModuleBase """ if not self.OutputFile: return S_ERROR( 'No output file defined' ) if self.isProdJob: if self.workflow_commons.has_key('ProductionOutputData'): outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_sim_") or obj.lower().count("_rec_") or obj.lower().count("_dst_"): self.OutputFile = os.path.basename(obj) else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. @return: S_OK() """ if self.step_commons.has_key('startFrom'): self.startFrom = self.step_commons['startFrom'] if self.step_commons.has_key('stdhepFile'): inputf = self.step_commons["stdhepFile"] if not type(inputf) == types.ListType: inputf = inputf.split(";") self.InputFile = inputf if self.step_commons.has_key("inputmacFile"): self.SteeringFile = self.step_commons['inputmacFile'] if self.step_commons.has_key('detectorModel'): self.detectorModel = self.step_commons['detectorModel'] if self.step_commons.has_key("RandomSeed"): self.randomseed = self.step_commons["RandomSeed"] ##Move below to ModuleBase as common to Mokka elif self.workflow_commons.has_key("IS_PROD"): self.randomseed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"]))) elif self.jobID: self.randomseed = self.jobID if self.workflow_commons.has_key("IS_PROD"): if self.workflow_commons["IS_PROD"]: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".stdhep") > -1 or files.lower().find(".hepevt") > -1: self.InputFile.append(files) break return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if self.WorkflowStartFrom: self.startFrom = self.WorkflowStartFrom self.randomSeed = self._determineRandomSeed() if "IS_PROD" in self.workflow_commons and self.workflow_commons["IS_PROD"]: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.endswith( DDSIMINPUTFORMATS ): self.InputFile.append(files) return S_OK('Parameters resolved')
def applicationSpecificInputs(self): if self.step_commons.has_key("NbOfEvts"): self.NumberOfEvents = self.step_commons["NbOfEvts"] else: return S_ERROR("Number of events to process not specified") if self.workflow_commons.has_key("IS_PROD"): if self.workflow_commons["IS_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if self.workflow_commons.has_key('ProductionOutputData'): outputlist = self.workflow_commons['ProductionOutputData'].split(";") baseoutputfile = self.OutputFile.split(".stdhep")[0] for obj in outputlist: if obj.count(baseoutputfile): self.OutputFile = os.path.basename(obj) else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) return S_OK()
def applicationSpecificInputs(self): if not self.OutputFile: dircont = os.listdir("./") for myfile in dircont: if myfile.count(".stdhep"): self.OutputFile = myfile.replace(".stdhep", "_reduced.stdhep") break if not self.OutputFile: return S_ERROR("Could not find suitable OutputFile name") if "IS_PROD" in self.workflow_commons and self.workflow_commons[ "IS_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons[ 'ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_gen_"): self.OutputFile = os.path.basename(obj) break else: self.OutputFile = getProdFilename( self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) self.log.notice("Outputfile: %s" % self.OutputFile) if self.inlineCuts: cfile = open("cuts_local.txt", "w") cfile.write("\n".join(self.inlineCuts.split(";"))) cfile.close() self.SteeringFile = "cuts_local.txt" return S_OK()
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if 'inputXML' in self.step_commons: self.SteeringFile = self.step_commons['inputXML'] #TODO: Next is necessary for old interface, should be removed when old prods are archived. if 'outputREC' in self.step_commons: self.outputREC = self.step_commons['outputREC'] if 'outputDST' in self.step_commons: self.outputDST = self.step_commons['outputDST'] if 'inputSlcio' in self.step_commons: inputf = self.step_commons["inputSlcio"] if not isinstance(inputf, list): inputf = inputf.split(";") self.InputFile = inputf #FIXME: hardcode default detector model add check for detectorModel when submitting lcsim jobs! #because currently no detectormodel is required! if "IS_PROD" in self.workflow_commons: if not self.detectorModel: self.detectorModel = "clic_sid_cdr.zip" if 'ExtraParams' in self.step_commons: self.extraparams = self.step_commons['ExtraParams'] if 'IS_PROD' in self.workflow_commons: if self.workflow_commons["IS_PROD"]: #self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #if 'MokkaOutput' in self.workflow_commons: # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons[ 'ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_rec_"): self.outputREC = os.path.basename(obj) elif obj.lower().count("_dst_"): self.outputDST = os.path.basename(obj) elif obj.lower().count("_sim_"): self.InputFile = [os.path.basename(obj)] else: self.outputREC = getProdFilename( self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) self.outputDST = getProdFilename( self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if 'SLICOutput' in self.workflow_commons: self.InputFile = [ getProdFilename( self.workflow_commons["SLICOutput"], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) ] if 'aliasproperties' in self.step_commons: self.aliasproperties = self.step_commons["aliasproperties"] if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".slcio") > -1: self.InputFile.append(files) self.log.info("Input files to treat %s" % self.InputFile) return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if self.WorkflowStartFrom: self.startFrom = self.WorkflowStartFrom if 'stdhepFile' in self.step_commons: inputf = self.step_commons["stdhepFile"] if not isinstance(inputf, list): inputf = inputf.split(";") self.InputFile = inputf self.RandomSeed = self.determineRandomSeed() resDBSlice = self.determineDBSlice() if not resDBSlice['OK']: return resDBSlice if "IS_PROD" in self.workflow_commons: if self.workflow_commons["IS_PROD"]: self.mcRunNumber = self.RandomSeed #self.OutputFile = getProdFilename(self.outputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons[ 'ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_sim_"): self.OutputFile = os.path.basename(obj) elif obj.lower().count("_gen_"): self.InputFile = [os.path.basename(obj)] else: self.OutputFile = getProdFilename( self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]), self.workflow_commons, ) #if 'WhizardOutput' in self.workflow_commons: # self.InputFile = getProdFilename(self.workflow_commons["WhizardOutput"], # int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) self.InputFile = [ getProdFilename( self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) ] if len(self.InputData): if 'EvtClass' in self.inputdataMeta and not self.ProcessID: self.ProcessID = self.inputdataMeta['EvtClass'] if 'EvtType' in self.inputdataMeta and not self.ProcessID: self.ProcessID = self.inputdataMeta['EvtType'] if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".stdhep") > -1 or files.lower().find( ".hepevt") > -1: self.InputFile.append(files) break return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. @return: S_OK() """ if self.step_commons.has_key('inputXML'): self.SteeringFile = self.step_commons['inputXML'] #TODO: Next is necessary for old interface, should be removed when old prods are archived. if self.step_commons.has_key('outputREC'): self.outputREC = self.step_commons['outputREC'] if self.step_commons.has_key('outputDST'): self.outputDST = self.step_commons['outputDST'] if self.step_commons.has_key("inputSlcio"): inputf = self.step_commons["inputSlcio"] if not type(inputf) == types.ListType: inputf = inputf.split(";") self.InputFile = inputf if self.step_commons.has_key('ExtraParams'): self.extraparams = self.step_commons['ExtraParams'] if self.workflow_commons.has_key("IS_PROD"): if self.workflow_commons["IS_PROD"]: #self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #if self.workflow_commons.has_key("MokkaOutput"): # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if self.workflow_commons.has_key('ProductionOutputData'): outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_rec_"): self.outputREC = os.path.basename(obj) elif obj.lower().count("_dst_"): self.outputDST = os.path.basename(obj) elif obj.lower().count("_sim_"): self.InputFile = [os.path.basename(obj)] else: self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if self.workflow_commons.has_key("SLICOutput"): self.InputFile = [getProdFilename(self.workflow_commons["SLICOutput"], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]))] if self.step_commons.has_key("aliasproperties"): self.aliasproperties = self.step_commons["aliasproperties"] if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".slcio") > -1: self.InputFile.append(files) self.log.info("Input files to treat %s" % self.InputFile) return S_OK('Parameters resolved')
def constructProductionLFNs(paramDict): """ Used for local testing of a workflow, a temporary measure until LFN construction is tidied. This works using the workflow commons for on the fly construction. :param dict paramDict: dictionary with at least the keys ``PRODUCTION_ID``, ``JOB_ID``, ``outputList`` :returns: S_OK with ``ProductionOutputData``, ``LogFilePath``, ``LogTargetPath`` """ result = checkForMandatoryKeys(paramDict, ['PRODUCTION_ID', 'JOB_ID', 'outputList']) if not result['OK']: return result productionID = paramDict['PRODUCTION_ID'] jobID = paramDict['JOB_ID'] outputList = paramDict['outputList'] fileTupleList = [] gLogger.verbose('outputList %s' % (outputList)) for info in outputList: #Nasty check on whether the created code parameters were not updated e.g. when changing defaults in a workflow fileName = info['outputFile'] #rename to take care of correct path fileName = getProdFilename(fileName, int(productionID), int(jobID)) fileTupleList.append((info['outputPath'], fileName)) #Get all LFN(s) to output data outputData = [] for fileTuple in fileTupleList: lfn = fileTuple[0] + "/" + str(productionID).zfill(8) + "/" + str( int(jobID) / 1000).zfill(3) + "/" + fileTuple[1] lfn = cleanUpLFNPath(lfn) outputData.append(lfn) #Get log file path - unique for all modules ## get First outputfile basePath = fileTupleList[0][0] #TODO adjust for ILD logPath = basePath + "/" + str(productionID).zfill(8) + "/LOG" #used for logFile upload to the LogSE logFilePath = [ cleanUpLFNPath('%s/%s' % (logPath, str(int(jobID) / 1000).zfill(3))) ] resLogs = getLogPath(dict(PRODUCTION_ID=productionID, JOB_ID=jobID, LogFilePath=logFilePath), basePath=basePath) if not resLogs['OK']: return resLogs logTargetPath = resLogs['Value']['LogTargetPath'] logFilePath = resLogs['Value']['LogFilePath'] if not outputData: gLogger.info('No output data LFN(s) constructed') else: gLogger.verbose('Created the following output data LFN(s):\n%s' % (string.join(outputData, '\n'))) gLogger.verbose('Log file path is:\n%s' % logFilePath[0]) gLogger.verbose('Log target path is:\n%s' % logTargetPath[0]) jobOutputs = { 'ProductionOutputData': outputData, 'LogFilePath': logFilePath, 'LogTargetPath': logTargetPath } return S_OK(jobOutputs)
def applicationSpecificInputs(self): """Resolve module input :return: S_OK() """ self.parameters['ENERGY'] = self.energy if not self.RandomSeed and self.jobID: self.RandomSeed = self.jobID if 'IS_PROD' in self.workflow_commons or 'IS_DBD_GEN_PROD' in self.workflow_commons: self.RandomSeed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"]))) self.parameters['SEED'] = self.RandomSeed self.parameters['NBEVTS'] = self.NumberOfEvents self.parameters['LUMI'] = self.Lumi ##EVER USED??? if 'SusyModel' in self.step_commons: self.susymodel = self.step_commons['SusyModel'] self.SteeringFile = os.path.basename(self.step_commons.get("InputFile", self.SteeringFile)) if self.SteeringFile == "whizard.in": os.rename(self.SteeringFile, "whizardnew.in") self.SteeringFile = "whizardnew.in" self.parameters['PROCESS'] = self.evttype listofparams = self.steeringparameters.split(";") for param in listofparams: if param.count("="): self.parameters[param.split("=")[0]] = param.split("=")[1] if self.OptionsDictStr: LOG.info("Will use whizard.in definition from WhizardOptions.") try: self.optionsdict = eval(self.OptionsDictStr) if 'integration_input' not in self.optionsdict: self.optionsdict['integration_input'] = {} if 'seed' not in self.optionsdict['integration_input']: self.optionsdict['integration_input']['seed'] = int(self.RandomSeed) if 'process_input' in self.optionsdict: if 'sqrts' in self.optionsdict['process_input']: self.energy = self.optionsdict['process_input']['sqrts'] except: return S_ERROR("Could not convert string to dictionary for optionsdict") if self.GenLevelCutDictStr: LOG.info("Found generator level cuts") try: self.genlevelcuts = eval(self.GenLevelCutDictStr) except: return S_ERROR("Could not convert the generator level cuts back to dictionary") if not len(self.SteeringFile) and not self.optionsdict: self.getProcessInFile = True if "IS_PROD" in self.workflow_commons: if self.workflow_commons["IS_PROD"] and not self.willCut: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_gen_"): self.OutputFile = os.path.basename(obj) break else: #This is because most likely there is stdhepcut running after self.OutputFile = "willcut.stdhep" #getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if "IS_DBD_GEN_PROD" in self.workflow_commons and self.workflow_commons["IS_DBD_GEN_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: self.OutputFile = os.path.basename(obj) break else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) return S_OK()
def applicationSpecificInputs(self): """ By convention the module parameters are resolved here. """ self.log.verbose("Workflow commons:") self.log.verbose(self.workflow_commons) self.log.verbose("Step commons:") self.log.verbose(self.step_commons) if self.step_commons.has_key('Enable'): self.enable = self.step_commons['Enable'] if not type(self.enable) == type(True): self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable) self.enable = False if self.step_commons.has_key('TestFailover'): self.enable = self.step_commons['TestFailover'] if not type(self.failoverTest) == type(True): self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest) self.failoverTest = False if self.workflow_commons.has_key("PRODUCTION_ID"): self.PRODUCTION_ID = self.workflow_commons["PRODUCTION_ID"] if os.environ.has_key('JOBID'): self.log.verbose('Found WMS JobID = %s' % self.jobID) else: self.log.info('No WMS JobID found, disabling module via control flag') self.enable = False if self.workflow_commons.has_key('Request'): self.request = self.workflow_commons['Request'] else: self.request = RequestContainer() self.request.setRequestName('job_%s_request.xml' % self.jobID) self.request.setJobID(self.jobID) self.request.setSourceComponent("Job_%s" % self.jobID) ##This is the thing that is used to establish the list of outpufiles to treat: ## Make sure that all that is in the : "listoutput" and also in the ProductionData ## is treated properly. Needed as whatever is in listoutput does not contain any reference to the ## prodID and task ID. Also if for some reason a step failed, then the corresponding data will not be there if self.workflow_commons.has_key('outputList'): self.outputList = self.workflow_commons['outputList'] if self.workflow_commons.has_key('ProductionOutputData'): proddata = self.workflow_commons['ProductionOutputData'].split(";") self.log.verbose("prod data : %s" % proddata ) olist = {} for obj in self.outputList: fname_in_outputlist = obj['outputFile'].lower() extension = '' if fname_in_outputlist.count("_sim") or fname_in_outputlist.count("_rec") or fname_in_outputlist.count("_dst"): extension = ".slcio" elif fname_in_outputlist.count("_gen"): extension = ".stdhep" fname_in_outputlist = fname_in_outputlist.replace(extension,"") for prodfile in proddata: prodfile = os.path.basename(prodfile) extension = '' if prodfile.count("_sim") or prodfile.count("_rec") or prodfile.count("_dst"): extension = ".slcio" elif prodfile.count("_gen"): extension = ".stdhep" prodfile = prodfile.replace(extension,"") if olist.has_key(prodfile): ## This has already been treated, no need to come back to it. continue appdict = {} if (fname_in_outputlist.count("_gen")):# and prodfile.lower().count("_gen_")) : genf = obj['outputFile'].split("_gen")[0] genf += "_gen" if (prodfile.count(genf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict if (fname_in_outputlist.count("_sim")): simf = obj['outputFile'].split("_sim")[0] simf += "_sim" if (prodfile.count(simf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict self.log.verbose('olist %s'%olist) if (fname_in_outputlist.count("_rec")): recf = obj['outputFile'].split("_rec")[0] recf += "_rec" if (prodfile.count(recf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict break if (fname_in_outputlist.count("_dst") and prodfile.lower().count("_dst_")): dstf = obj['outputFile'].split("_dst")[0] dstf += "_dst" if (prodfile.count(dstf)): appdict.update(obj) appdict['outputFile'] = prodfile+extension olist[prodfile] = appdict break self.outputList = olist.values() else: olist = [] for obj in self.outputList: appdict = obj appdict['outputFile'] = getProdFilename(obj['outputFile'], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) olist.append(appdict) self.outputList = olist self.log.verbose("OutputList : %s" % self.outputList) if self.workflow_commons.has_key('outputMode'): self.outputMode = self.workflow_commons['outputMode'] if self.workflow_commons.has_key('outputDataFileMask'): self.outputDataFileMask = self.workflow_commons['outputDataFileMask'] if not type(self.outputDataFileMask) == type([]): self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')] #result = constructProductionLFNs(self.workflow_commons) #if not result['OK']: # self.log.error('Could not create production LFNs',result['Message']) # return result #self.prodOutputLFNs=result['Value']['ProductionOutputData'] if self.workflow_commons.has_key('ProductionOutputData'): self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";") else: self.prodOutputLFNs = [] return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ By convention the module parameters are resolved here. """ self.log.debug("Workflow commons: %s" % self.workflow_commons) self.enable = self.step_commons.get('Enable', self.enable) if not isinstance( self.enable, bool ): self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable) self.enable = False self.failoverTest = self.step_commons.get('TestFailover', self.failoverTest) if not isinstance( self.failoverTest, bool ): self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest) self.failoverTest = False self.productionID = self.workflow_commons.get("PRODUCTION_ID", self.productionID) self.jobID = os.environ.get('JOBID', self.jobID) if self.jobID: self.log.verbose('Found WMS JobID = %s' % self.jobID) else: self.log.info('No WMS JobID found, disabling module via control flag') self.enable = False ##This is the thing that is used to establish the list of outpufiles to treat: ## Make sure that all that is in the : "outputList" and also in the ProductionData ## is treated properly. Needed as whatever is in outputList does not contain any reference to the ## prodID and task ID. Also if for some reason a step failed, then the corresponding data will not be there self.outputList = self.workflow_commons.get('outputList', self.outputList) if self.outputList: if 'ProductionOutputData' in self.workflow_commons: productionData = self.workflow_commons['ProductionOutputData'].split(";") self.log.verbose("prod data : %s" % productionData ) treatedOutputlist = {} for expectedOutputfile in self.outputList: self.log.debug("Treating file: %s" % expectedOutputfile['outputFile']) self.getTreatedOutputlistNew(productionData, treatedOutputlist, expectedOutputfile) self.outputList = treatedOutputlist.values() else: olist = [] for expectedOutputfile in self.outputList: appdict = expectedOutputfile appdict['outputFile'] = getProdFilename(expectedOutputfile['outputFile'], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) olist.append(appdict) self.outputList = olist self.log.verbose("OutputList : %s" % self.outputList) self.outputMode = self.workflow_commons.get('outputMode', self.outputMode) self.outputDataFileMask = self.workflow_commons.get('outputDataFileMask', self.outputDataFileMask) if not isinstance( self.outputDataFileMask, list ): self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')] #result = constructProductionLFNs(self.workflow_commons) #if not result['OK']: # self.log.error('Could not create production LFNs',result['Message']) # return result #self.prodOutputLFNs=result['Value']['ProductionOutputData'] tempOutputLFNs = self.workflow_commons.get('ProductionOutputData', self.prodOutputLFNs) if isinstance( tempOutputLFNs, basestring ): self.prodOutputLFNs = tempOutputLFNs.split(";") else: self.prodOutputLFNs = tempOutputLFNs return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. @return: S_OK() """ if self.step_commons.has_key('numberOfEvents'): self.NumberOfEvents = self.step_commons['numberOfEvents'] if self.step_commons.has_key('startFrom'): self.startFrom = self.step_commons['startFrom'] #Need to keep until old prods are archived. if self.step_commons.has_key("steeringFile"): self.SteeringFile = self.step_commons['steeringFile'] if self.step_commons.has_key('stdhepFile'): inputf = self.step_commons["stdhepFile"] if not type(inputf) == types.ListType: inputf = inputf.split(";") self.InputFile = inputf if self.step_commons.has_key('macFile'): self.macFile = self.step_commons['macFile'] if self.step_commons.has_key('detectorModel'): self.detectorModel = self.step_commons['detectorModel'] if self.step_commons.has_key('ProcessID'): self.processID = self.step_commons['ProcessID'] if not self.RandomSeed: if self.step_commons.has_key("RandomSeed"): self.RandomSeed = self.step_commons["RandomSeed"] elif self.jobID: self.RandomSeed = self.jobID if self.workflow_commons.has_key("IS_PROD"): self.RandomSeed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"]))) if self.step_commons.has_key('dbSlice'): self.dbSlice = self.step_commons['dbSlice'] if self.workflow_commons.has_key("IS_PROD"): if self.workflow_commons["IS_PROD"]: self.mcRunNumber = self.RandomSeed #self.OutputFile = getProdFilename(self.outputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if self.workflow_commons.has_key('ProductionOutputData'): outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_sim_"): self.OutputFile = os.path.basename(obj) elif obj.lower().count("_gen_"): self.InputFile = [os.path.basename(obj)] else: self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) #if self.workflow_commons.has_key("WhizardOutput"): # self.InputFile = getProdFilename(self.workflow_commons["WhizardOutput"], # int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]))] if len(self.InputData): if 'EvtClass' in self.inputdataMeta and not self.processID: self.processID = self.inputdataMeta['EvtClass'] if 'EvtType' in self.inputdataMeta and not self.processID: self.processID = self.inputdataMeta['EvtType'] if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".stdhep") > -1 or files.lower().find(".hepevt") > -1: self.InputFile.append(files) break return S_OK('Parameters resolved')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. @return: S_OK() """ ##TODO: Need to keep for old interface. Move to ModuleBase if self.step_commons.has_key('inputSlcio'): inputf = self.step_commons["inputSlcio"] if not type(inputf) == types.ListType: if len(inputf): inputf = inputf.split(";") else: inputf = [] self.InputFile = inputf if self.workflow_commons.has_key('ParametricInputSandbox'): paramsb = self.workflow_commons['ParametricInputSandbox'] if not type(paramsb) == types.ListType: if len(paramsb): paramsb = paramsb.split(";") else: paramsb = [] self.InputFile += paramsb if self.step_commons.has_key('inputXML'): self.SteeringFile = self.step_commons['inputXML'] if self.step_commons.has_key('inputGEAR'): self.inputGEAR = self.step_commons['inputGEAR'] if self.step_commons.has_key('EvtsToProcess'): self.NumberOfEvents = self.step_commons['EvtsToProcess'] ##Backward compat needed, cannot remove yet. if self.step_commons.has_key('outputREC'): self.outputREC = self.step_commons['outputREC'] if self.step_commons.has_key('outputDST'): self.outputDST = self.step_commons['outputDST'] if self.workflow_commons.has_key("IS_PROD"): if self.workflow_commons["IS_PROD"] and len(self.OutputFile)==0: #self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #if self.workflow_commons.has_key("MokkaOutput"): # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if self.workflow_commons.has_key('ProductionOutputData'): outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_rec_"): self.outputREC = os.path.basename(obj) elif obj.lower().count("_dst_"): self.outputDST = os.path.basename(obj) elif obj.lower().count("_sim_"): self.InputFile = [os.path.basename(obj)] else: self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) #if self.workflow_commons.has_key("MokkaOutput"): # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]))] if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".slcio") > -1: self.InputFile.append(files) return S_OK('Parameters resolved')
def test_getnames(self): """test ResolvePathsAndNames getNames...........................................................""" res = getProdFilename("toto_gen.stdhep", 12345,123) self.assertEqual(res,'toto_gen_12345_123.stdhep')
def applicationSpecificInputs(self): """Resolve module input :return: S_OK() """ self.parameters['ENERGY'] = self.energy if not self.RandomSeed and self.jobID: self.RandomSeed = self.jobID if 'IS_PROD' in self.workflow_commons or 'IS_DBD_GEN_PROD' in self.workflow_commons: self.RandomSeed = int( str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"]))) self.parameters['SEED'] = self.RandomSeed self.parameters['NBEVTS'] = self.NumberOfEvents self.parameters['LUMI'] = self.Lumi ##EVER USED??? if 'SusyModel' in self.step_commons: self.susymodel = self.step_commons['SusyModel'] self.SteeringFile = os.path.basename( self.step_commons.get("InputFile", self.SteeringFile)) if self.SteeringFile == "whizard.in": os.rename(self.SteeringFile, "whizardnew.in") self.SteeringFile = "whizardnew.in" self.parameters['PROCESS'] = self.evttype listofparams = self.steeringparameters.split(";") for param in listofparams: if param.count("="): self.parameters[param.split("=")[0]] = param.split("=")[1] if self.OptionsDictStr: self.log.info( "Will use whizard.in definition from WhizardOptions.") try: self.optionsdict = eval(self.OptionsDictStr) if 'integration_input' not in self.optionsdict: self.optionsdict['integration_input'] = {} if 'seed' not in self.optionsdict['integration_input']: self.optionsdict['integration_input']['seed'] = int( self.RandomSeed) if 'process_input' in self.optionsdict: if 'sqrts' in self.optionsdict['process_input']: self.energy = self.optionsdict['process_input'][ 'sqrts'] except: return S_ERROR( "Could not convert string to dictionary for optionsdict") if self.GenLevelCutDictStr: self.log.info("Found generator level cuts") try: self.genlevelcuts = eval(self.GenLevelCutDictStr) except: return S_ERROR( "Could not convert the generator level cuts back to dictionary" ) if not len(self.SteeringFile) and not self.optionsdict: self.getProcessInFile = True if "IS_PROD" in self.workflow_commons: if self.workflow_commons["IS_PROD"] and not self.willCut: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons[ 'ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_gen_"): self.OutputFile = os.path.basename(obj) break else: #This is because most likely there is stdhepcut running after self.OutputFile = "willcut.stdhep" #getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if "IS_DBD_GEN_PROD" in self.workflow_commons and self.workflow_commons[ "IS_DBD_GEN_PROD"]: #self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons[ 'ProductionOutputData'].split(";") for obj in outputlist: self.OutputFile = os.path.basename(obj) break else: self.OutputFile = getProdFilename( self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) return S_OK()
def constructProductionLFNs(paramDict): """ Used for local testing of a workflow, a temporary measure until LFN construction is tidied. This works using the workflow commons for on the fly construction. """ keys = ['PRODUCTION_ID', 'JOB_ID', 'JobType', 'outputList'] for k in keys: if not paramDict.has_key(k): return S_ERROR('%s not defined' % k) productionID = paramDict['PRODUCTION_ID'] jobID = paramDict['JOB_ID'] # wfMode = paramDict['dataType'] #wfLfnprefix=paramDict['lfnprefix'] #wfLfnpostfix=paramDict['lfnpostfix'] wfMask = "" # wfMask = paramDict['outputDataFileMask'] if not type(wfMask) == type([]): wfMask = [i.lower().strip() for i in wfMask.split(';')] wfType = paramDict['JobType'] outputList = paramDict['outputList'] inputData = '' if paramDict.has_key('InputData'): inputData = paramDict['InputData'] res = getVOfromProxyGroup() #res = gConfig.getOption("/DIRAC/VirtualOrganization", "ilc") if not res['OK']: gLogger.error('Could not get VO from CS, assuming ilc') vo = 'ilc' else: vo = res['Value'] fileTupleList = [] #gLogger.verbose('wfLfnprefix = %s, wfLfnpostfix = %s, wfMask = %s, wfType=%s' %(wfLfnprefix,wfLfnpostfix,wfMask,wfType)) gLogger.verbose('outputList %s' % (outputList)) for info in outputList: #Nasty check on whether the created code parameters were not updated e.g. when changing defaults in a workflow fileName = info['outputFile'] #rename to take care of correct path fileName = getProdFilename(fileName, int(productionID), int(jobID)) #index=0 #if not re.search('^\d',fileName[index]): # index+=1 #if not fileName[index]==str(productionID).zfill(8): # fileName[index]=str(productionID).zfill(8) #if not fileName[index+1]==str(jobID).zfill(8): # fileName[index+1]=str(jobID).zfill(8) fileTupleList.append((info['outputPath'], fileName)) lfnRoot = '' debugRoot = '' #if inputData: # gLogger.verbose('Making LFN_ROOT for job with inputdata: %s' %(inputData)) # lfnRoot = _getLFNRoot(inputData,wfLfnpostfix) # debugRoot= _getLFNRoot('','debug',wfLfnpostfix) #else: # lfnRoot = _getLFNRoot('',wfLfnprefix,wfLfnpostfix) # gLogger.verbose('LFN_ROOT is: %s' %(lfnRoot)) # debugRoot= _getLFNRoot('','debug',wfLfnpostfix) #lfnRoot = #gLogger.verbose('LFN_ROOT is: %s' %(lfnRoot)) #if not lfnRoot: # return S_ERROR('LFN root could not be constructed') #Get all LFN(s) to both output data and BK lists at this point (fine for BK) outputData = [] #bkLFNs = [] debugLFNs = [] for fileTuple in fileTupleList: #lfn = _makeProductionLfn(str(jobID).zfill(8),lfnRoot,fileTuple,wfLfnprefix,str(productionID).zfill(8)) lfn = fileTuple[0] + "/" + str(productionID).zfill(8) + "/" + str(int(jobID)/1000).zfill(3) + "/" + fileTuple[1] if lfn.count('//'): lfn = lfn.replace('//','/') outputData.append(lfn) #bkLFNs.append(lfn) if debugRoot: #debugLFNs.append(_makeProductionLfn(str(jobID).zfill(8),debugRoot,fileTuple,wfLfnprefix,str(productionID).zfill(8))) debugLFNs.append("/" + vo + "/prod/debug/" + str(productionID).zfill(8)) #if debugRoot: # debugLFNs.append(_makeProductionLfn(str(jobID).zfill(8),debugRoot,('%s_core' % str(jobID).zfill(8) ,'core'),wfLfnprefix,str(productionID).zfill(8))) #Get log file path - unique for all modules #logPath = _makeProductionPath(str(jobID).zfill(8),lfnRoot,'LOG',wfLfnprefix,str(productionID).zfill(8),log=True) logPathtemp = fileTupleList[0][0].split("/") logPathroot = string.join(logPathtemp[0:len(logPathtemp)-1], "/") #TODO adjust for ILD logPath = logPathroot + "/LOG/" + str(productionID).zfill(8) logFilePath = ['%s/%s' % (logPath, str(int(jobID)/1000).zfill(3))] logTargetPath = ['%s/%s_%s.tar' % (logPath, str(productionID).zfill(8), str(int(jobID)).zfill(3))] #[ aside, why does makeProductionPath not append the jobID itself ???? # this is really only used in one place since the logTargetPath is just written to a text file (should be reviewed)... ] #Strip output data according to file mask if wfMask: newOutputData = [] #newBKLFNs = [] for od in outputData: for i in wfMask: if re.search('.%s$' % i, od): if not od in newOutputData: newOutputData.append(od) #for bk in bkLFNs: # newBKLFNs.append(bk) outputData = newOutputData #bkLFNs = newBKLFNs if not outputData: gLogger.info('No output data LFN(s) constructed') else: gLogger.verbose('Created the following output data LFN(s):\n%s' % (string.join(outputData,'\n'))) gLogger.verbose('Log file path is:\n%s' % logFilePath[0]) gLogger.verbose('Log target path is:\n%s' % logTargetPath[0]) #if bkLFNs: # gLogger.verbose('BookkeepingLFN(s) are:\n%s' %(string.join(bkLFNs,'\n'))) if debugLFNs: gLogger.verbose('DebugLFN(s) are:\n%s' % (string.join(debugLFNs, '\n'))) jobOutputs = {'ProductionOutputData' : outputData, 'LogFilePath' : logFilePath, 'LogTargetPath' : logTargetPath, 'DebugLFNs' : debugLFNs} return S_OK(jobOutputs)
def test_getnames(self): """test ResolvePathsAndNames getNames...........................................................""" res = getProdFilename("toto_gen.stdhep", 12345, 123) self.assertEqual(res, 'toto_gen_12345_123.stdhep')
def applicationSpecificInputs(self): """ Resolve all input variables for the module here. :return: S_OK() """ if 'inputXML' in self.step_commons: self.SteeringFile = self.step_commons['inputXML'] #TODO: Next is necessary for old interface, should be removed when old prods are archived. if 'outputREC' in self.step_commons: self.outputREC = self.step_commons['outputREC'] if 'outputDST' in self.step_commons: self.outputDST = self.step_commons['outputDST'] if 'inputSlcio' in self.step_commons: inputf = self.step_commons["inputSlcio"] if not isinstance( inputf, list ): inputf = inputf.split(";") self.InputFile = inputf #FIXME: hardcode default detector model add check for detectorModel when submitting lcsim jobs! #because currently no detectormodel is required! if "IS_PROD" in self.workflow_commons: if not self.detectorModel: self.detectorModel = "clic_sid_cdr.zip" if 'ExtraParams' in self.step_commons: self.extraparams = self.step_commons['ExtraParams'] if 'IS_PROD' in self.workflow_commons: if self.workflow_commons["IS_PROD"]: #self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) #if 'MokkaOutput' in self.workflow_commons: # self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]), # int(self.workflow_commons["JOB_ID"])) if 'ProductionOutputData' in self.workflow_commons: outputlist = self.workflow_commons['ProductionOutputData'].split(";") for obj in outputlist: if obj.lower().count("_rec_"): self.outputREC = os.path.basename(obj) elif obj.lower().count("_dst_"): self.outputDST = os.path.basename(obj) elif obj.lower().count("_sim_"): self.InputFile = [os.path.basename(obj)] else: self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"])) if 'SLICOutput' in self.workflow_commons: self.InputFile = [getProdFilename(self.workflow_commons["SLICOutput"], int(self.workflow_commons["PRODUCTION_ID"]), int(self.workflow_commons["JOB_ID"]))] if 'aliasproperties' in self.step_commons: self.aliasproperties = self.step_commons["aliasproperties"] if not len(self.InputFile) and len(self.InputData): for files in self.InputData: if files.lower().find(".slcio") > -1: self.InputFile.append(files) LOG.info("Input files to treat %s" % self.InputFile) return S_OK('Parameters resolved')