def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if self.WorkflowStartFrom:
self.startFrom = self.WorkflowStartFrom
if 'stdhepFile' in self.step_commons:
inputf = self.step_commons["stdhepFile"]
if not isinstance( inputf, list ):
inputf = inputf.split(";")
self.InputFile = inputf
self.RandomSeed = self.determineRandomSeed()
resDBSlice = self.determineDBSlice()
if not resDBSlice['OK']:
return resDBSlice
if "IS_PROD" in self.workflow_commons:
if self.workflow_commons["IS_PROD"]:
self.mcRunNumber = self.RandomSeed
#self.OutputFile = getProdFilename(self.outputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_sim_"):
self.OutputFile = os.path.basename(obj)
elif obj.lower().count("_gen_"):
self.InputFile = [os.path.basename(obj)]
else:
self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]),
self.workflow_commons,
)
#if 'WhizardOutput' in self.workflow_commons:
# self.InputFile = getProdFilename(self.workflow_commons["WhizardOutput"],
# int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))]
if len(self.InputData):
if 'EvtClass' in self.inputdataMeta and not self.ProcessID:
self.ProcessID = self.inputdataMeta['EvtClass']
if 'EvtType' in self.inputdataMeta and not self.ProcessID:
self.ProcessID = self.inputdataMeta['EvtType']
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".stdhep") > -1 or files.lower().find(".hepevt") > -1:
self.InputFile.append(files)
break
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if 'ParametricInputSandbox' in self.workflow_commons:
paramsb = self.workflow_commons['ParametricInputSandbox']
if not isinstance( paramsb, list ):
if paramsb:
paramsb = paramsb.split(";")
else:
paramsb = []
self.InputFile += paramsb
##Backward compat needed, cannot remove yet.
self.outputREC = self.step_commons.get('outputREC', self.outputREC)
self.outputDST = self.step_commons.get('outputDST', self.outputDST)
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"] and not self.OutputFile:
#self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#if 'MokkaOutput' in self.workflow_commons:
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_rec_"):
self.outputREC = os.path.basename(obj)
elif obj.lower().count("_dst_"):
self.outputDST = os.path.basename(obj)
elif obj.lower().count("_sim_"):
self.InputFile = [os.path.basename(obj)]
else:
self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
#if 'MokkaOutput' in self.workflow_commons:
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))]
if not self.InputFile and self.InputData:
for files in self.InputData:
if files.lower().find(".slcio") > -1:
self.InputFile.append(files)
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if self.workflow_commons.has_key('ParametricInputSandbox'):
paramsb = self.workflow_commons['ParametricInputSandbox']
if not type(paramsb) == types.ListType:
if len(paramsb):
paramsb = paramsb.split(";")
else:
paramsb = []
self.InputFile += paramsb
##Backward compat needed, cannot remove yet.
self.outputREC = self.step_commons.get('outputREC', self.outputREC)
self.outputDST = self.step_commons.get('outputDST', self.outputDST)
if self.workflow_commons.has_key("IS_PROD"):
if self.workflow_commons["IS_PROD"] and len(self.OutputFile)==0:
#self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#if self.workflow_commons.has_key("MokkaOutput"):
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if self.workflow_commons.has_key('ProductionOutputData'):
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_rec_"):
self.outputREC = os.path.basename(obj)
elif obj.lower().count("_dst_"):
self.outputDST = os.path.basename(obj)
elif obj.lower().count("_sim_"):
self.InputFile = [os.path.basename(obj)]
else:
self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
#if self.workflow_commons.has_key("MokkaOutput"):
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))]
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".slcio") > -1:
self.InputFile.append(files)
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Called from ModuleBase
"""
if 'NbEvtsKept' in self.step_commons:
self.NbEvtsKept = self.step_commons['NbEvtsKept']
if not self.NbEvtsKept:
return S_ERROR('Nb of events to keep MUST be specified')
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons[
'ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_gen_"):
self.InputFile = [os.path.basename(obj)]
self.OutputFile = self.InputFile[0]
else:
if 'WhizardOutput' in self.workflow_commons:
self.stdhepFile = getProdFilename(
self.workflow_commons["WhizardOutput"],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().count(".stdhep"):
self.InputFile.append(files)
break
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve LCIO concatenate specific parameters, called from ModuleBase
"""
if not self.OutputFile:
return S_ERROR( 'No output file defined' )
if self.workflow_commons.has_key("IS_PROD"):
if self.workflow_commons["IS_PROD"]:
if self.workflow_commons.has_key('ProductionOutputData'):
self.prod_outputdata = self.workflow_commons['ProductionOutputData'].split(";")
for obj in self.prod_outputdata:
if obj.lower().count("_gen_"):
self.OutputFile = os.path.basename(obj)
else:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".stdhep") > -1:
self.InputFile.append(files)
if self.step_commons.has_key('listoutput'):
if len(self.step_commons['listoutput']):
self.listoutput = self.step_commons['listoutput'][0]
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
if self.step_commons.has_key('CutFile'):
self.SteeringFile = self.step_commons['CutFile']
if self.step_commons.has_key('MaxNbEvts'):
self.MaxNbEvts = self.step_commons['MaxNbEvts']
if not self.OutputFile:
dircont = os.listdir("./")
for myfile in dircont:
if myfile.count(".stdhep"):
self.OutputFile = myfile.rstrip(".stdhep") + "_reduced.stdhep"
break
if not self.OutputFile:
return S_ERROR("Could not find suitable OutputFile name")
if self.workflow_commons.has_key("IS_PROD"):
if self.workflow_commons["IS_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if self.workflow_commons.has_key('ProductionOutputData'):
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_gen_"):
self.OutputFile = os.path.basename(obj)
break
else:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
return S_OK()
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if self.WorkflowStartFrom:
self.startFrom = self.WorkflowStartFrom
##Move below to ModuleBase as common to Mokka
if "IS_PROD" in self.workflow_commons:
self.RandomSeed = int(
str(int(self.workflow_commons["PRODUCTION_ID"])) +
str(int(self.workflow_commons["JOB_ID"])))
elif self.jobID:
self.RandomSeed = self.jobID
if 'IS_PROD' in self.workflow_commons and self.workflow_commons[
"IS_PROD"]:
self.OutputFile = getProdFilename(
self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".stdhep") > -1 or files.lower().find(
".hepevt") > -1:
self.InputFile.append(files)
break
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Called from ModuleBase
"""
if 'NbEvtsKept' in self.step_commons:
self.NbEvtsKept = self.step_commons['NbEvtsKept']
if not self.NbEvtsKept:
return S_ERROR('Nb of events to keep MUST be specified')
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_gen_"):
self.InputFile = [os.path.basename(obj)]
self.OutputFile = self.InputFile[0]
else:
if 'WhizardOutput' in self.workflow_commons:
self.stdhepFile = getProdFilename(self.workflow_commons["WhizardOutput"],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().count(".stdhep"):
self.InputFile.append(files)
break
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve LCIO concatenate specific parameters, called from ModuleBase
"""
if not self.OutputFile:
return S_ERROR('No output file defined')
if 'IS_PROD' in self.workflow_commons and self.workflow_commons[
"IS_PROD"]:
if 'ProductionOutputData' in self.workflow_commons:
self.prod_outputdata = self.workflow_commons[
'ProductionOutputData'].split(";")
for obj in self.prod_outputdata:
if obj.lower().count("_gen_"):
self.OutputFile = os.path.basename(obj)
else:
self.OutputFile = getProdFilename(
self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".stdhep") > -1:
self.InputFile.append(files)
if 'listoutput' in self.step_commons:
if len(self.step_commons['listoutput']):
self.listoutput = self.step_commons['listoutput'][0]
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if self.WorkflowStartFrom:
self.startFrom = self.WorkflowStartFrom
##Move below to ModuleBase as common to Mokka
if "IS_PROD" in self.workflow_commons:
self.RandomSeed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"])))
elif self.jobID:
self.RandomSeed = self.jobID
if 'IS_PROD' in self.workflow_commons and self.workflow_commons["IS_PROD"]:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".stdhep") > -1 or files.lower().find(".hepevt") > -1:
self.InputFile.append(files)
break
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if self.WorkflowStartFrom:
self.startFrom = self.WorkflowStartFrom
self.randomSeed = self._determineRandomSeed()
if "IS_PROD" in self.workflow_commons and self.workflow_commons[
"IS_PROD"]:
self.OutputFile = getProdFilename(
self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]),
self.workflow_commons,
)
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.endswith(DDSIMINPUTFORMATS):
self.InputFile.append(files)
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
if not self.OutputFile:
dircont = os.listdir("./")
for myfile in dircont:
if myfile.count(".stdhep"):
self.OutputFile = myfile.replace(".stdhep", "_reduced.stdhep")
break
if not self.OutputFile:
return S_ERROR("Could not find suitable OutputFile name")
if "IS_PROD" in self.workflow_commons and self.workflow_commons["IS_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_gen_"):
self.OutputFile = os.path.basename(obj)
break
else:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.log.notice("Outputfile: %s" % self.OutputFile)
if self.inlineCuts:
cfile = open("cuts_local.txt", "w")
cfile.write("\n".join(self.inlineCuts.split(";")))
cfile.close()
self.SteeringFile = "cuts_local.txt"
return S_OK()
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if 'ParametricInputSandbox' in self.workflow_commons:
paramsb = self.workflow_commons['ParametricInputSandbox']
if not isinstance( paramsb, list ):
if paramsb:
paramsb = paramsb.split(";")
else:
paramsb = []
self.InputFile += paramsb
##Backward compat needed, cannot remove yet.
self.outputREC = self.step_commons.get('outputREC', self.outputREC)
self.outputDST = self.step_commons.get('outputDST', self.outputDST)
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"] and not self.OutputFile:
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
LOG.info('Found ProductionOutputData in WF Commons', str(outputlist))
for obj in outputlist:
if obj.lower().count("_rec_"):
self.outputREC = os.path.basename(obj)
elif obj.lower().count("_dst_"):
self.outputDST = os.path.basename(obj)
elif obj.lower().count("_sim_"):
self.InputFile = [os.path.basename(obj)]
else:
prodID = int(self.workflow_commons['PRODUCTION_ID'])
jobID = int(self.workflow_commons["JOB_ID"])
self.outputREC = getProdFilename(self.outputREC, prodID, jobID)
self.outputDST = getProdFilename(self.outputDST, prodID, jobID)
self.InputFile = [getProdFilename(self.InputFile, prodID, jobID)]
LOG.info('OutputDST:', self.outputDST)
LOG.info('OutputREC:', self.outputREC)
if not self.InputFile and self.InputData:
for files in self.InputData:
if files.lower().find(".slcio") > -1:
self.InputFile.append(files)
return S_OK('Parameters resolved')
def constructProductionLFNs(paramDict):
""" Used for local testing of a workflow, a temporary measure until
LFN construction is tidied. This works using the workflow commons for
on the fly construction.
:param dict paramDict: dictionary with at least the keys ``PRODUCTION_ID``, ``JOB_ID``, ``outputList``
:returns: S_OK with ``ProductionOutputData``, ``LogFilePath``, ``LogTargetPath``
"""
result = checkForMandatoryKeys(paramDict, ['PRODUCTION_ID', 'JOB_ID', 'outputList'])
if not result['OK']:
return result
productionID = paramDict['PRODUCTION_ID']
jobID = paramDict['JOB_ID']
outputList = paramDict['outputList']
fileTupleList = []
gLogger.verbose('outputList %s' % (outputList))
for info in outputList:
#Nasty check on whether the created code parameters were not updated e.g. when changing defaults in a workflow
fileName = info['outputFile']
#rename to take care of correct path
fileName = getProdFilename(fileName, int(productionID), int(jobID))
fileTupleList.append((info['outputPath'], fileName))
#Get all LFN(s) to output data
outputData = []
for fileTuple in fileTupleList:
lfn = fileTuple[0] + "/" + str(productionID).zfill(8) + "/" + str(int(jobID)/1000).zfill(3) + "/" + fileTuple[1]
lfn = cleanUpLFNPath(lfn)
outputData.append(lfn)
#Get log file path - unique for all modules
## get First outputfile
basePath = fileTupleList[0][0]
#TODO adjust for ILD
logPath = basePath + "/" + str(productionID).zfill(8) + "/LOG"
#used for logFile upload to the LogSE
logFilePath = [cleanUpLFNPath('%s/%s' % (logPath, str(int(jobID)/1000).zfill(3)))]
resLogs = getLogPath( dict(PRODUCTION_ID=productionID,
JOB_ID=jobID,
LogFilePath=logFilePath), basePath=basePath )
if not resLogs['OK']:
return resLogs
logTargetPath = resLogs['Value']['LogTargetPath']
logFilePath = resLogs['Value']['LogFilePath']
if not outputData:
gLogger.info('No output data LFN(s) constructed')
else:
gLogger.verbose('Created the following output data LFN(s):\n%s' % (string.join(outputData,'\n')))
gLogger.verbose('Log file path is:\n%s' % logFilePath[0])
gLogger.verbose('Log target path is:\n%s' % logTargetPath[0])
jobOutputs = {'ProductionOutputData' : outputData, 'LogFilePath' : logFilePath, 'LogTargetPath' : logTargetPath}
return S_OK(jobOutputs)
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
self.randomSeed = self._determineRandomSeed()
if "IS_PROD" in self.workflow_commons and self.workflow_commons["IS_PROD"]:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]),
self.workflow_commons,
)
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
self.randomSeed = self._determineRandomSeed()
if "IS_PROD" in self.workflow_commons and self.workflow_commons[
"IS_PROD"]:
self.OutputFile = getProdFilename(
self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]),
self.workflow_commons,
)
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
baseoutputfile = self.OutputFile.split(".stdhep")[0]
for obj in outputlist:
if obj.count(baseoutputfile):
self.OutputFile = os.path.basename(obj)
else:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
return S_OK()
def applicationSpecificInputs(self):
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
baseoutputfile = self.OutputFile.split(".stdhep")[0]
for obj in outputlist:
if obj.count(baseoutputfile):
self.OutputFile = os.path.basename(obj)
else:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
return S_OK()
def applicationSpecificInputs(self):
""" Resolve LCIO concatenate specific parameters, called from ModuleBase
"""
if not self.OutputFile:
return S_ERROR( 'No output file defined' )
if self.isProdJob:
if self.workflow_commons.has_key('ProductionOutputData'):
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_sim_") or obj.lower().count("_rec_") or obj.lower().count("_dst_"):
self.OutputFile = os.path.basename(obj)
else:
self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
@return: S_OK()
"""
if self.step_commons.has_key('startFrom'):
self.startFrom = self.step_commons['startFrom']
if self.step_commons.has_key('stdhepFile'):
inputf = self.step_commons["stdhepFile"]
if not type(inputf) == types.ListType:
inputf = inputf.split(";")
self.InputFile = inputf
if self.step_commons.has_key("inputmacFile"):
self.SteeringFile = self.step_commons['inputmacFile']
if self.step_commons.has_key('detectorModel'):
self.detectorModel = self.step_commons['detectorModel']
if self.step_commons.has_key("RandomSeed"):
self.randomseed = self.step_commons["RandomSeed"]
##Move below to ModuleBase as common to Mokka
elif self.workflow_commons.has_key("IS_PROD"):
self.randomseed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"])))
elif self.jobID:
self.randomseed = self.jobID
if self.workflow_commons.has_key("IS_PROD"):
if self.workflow_commons["IS_PROD"]:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".stdhep") > -1 or files.lower().find(".hepevt") > -1:
self.InputFile.append(files)
break
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if self.WorkflowStartFrom:
self.startFrom = self.WorkflowStartFrom
self.randomSeed = self._determineRandomSeed()
if "IS_PROD" in self.workflow_commons and self.workflow_commons["IS_PROD"]:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.endswith( DDSIMINPUTFORMATS ):
self.InputFile.append(files)
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
if self.step_commons.has_key("NbOfEvts"):
self.NumberOfEvents = self.step_commons["NbOfEvts"]
else:
return S_ERROR("Number of events to process not specified")
if self.workflow_commons.has_key("IS_PROD"):
if self.workflow_commons["IS_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if self.workflow_commons.has_key('ProductionOutputData'):
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
baseoutputfile = self.OutputFile.split(".stdhep")[0]
for obj in outputlist:
if obj.count(baseoutputfile):
self.OutputFile = os.path.basename(obj)
else:
self.OutputFile = getProdFilename(self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
return S_OK()
def applicationSpecificInputs(self):
if not self.OutputFile:
dircont = os.listdir("./")
for myfile in dircont:
if myfile.count(".stdhep"):
self.OutputFile = myfile.replace(".stdhep",
"_reduced.stdhep")
break
if not self.OutputFile:
return S_ERROR("Could not find suitable OutputFile name")
if "IS_PROD" in self.workflow_commons and self.workflow_commons[
"IS_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons[
'ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_gen_"):
self.OutputFile = os.path.basename(obj)
break
else:
self.OutputFile = getProdFilename(
self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.log.notice("Outputfile: %s" % self.OutputFile)
if self.inlineCuts:
cfile = open("cuts_local.txt", "w")
cfile.write("\n".join(self.inlineCuts.split(";")))
cfile.close()
self.SteeringFile = "cuts_local.txt"
return S_OK()
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if 'inputXML' in self.step_commons:
self.SteeringFile = self.step_commons['inputXML']
#TODO: Next is necessary for old interface, should be removed when old prods are archived.
if 'outputREC' in self.step_commons:
self.outputREC = self.step_commons['outputREC']
if 'outputDST' in self.step_commons:
self.outputDST = self.step_commons['outputDST']
if 'inputSlcio' in self.step_commons:
inputf = self.step_commons["inputSlcio"]
if not isinstance(inputf, list):
inputf = inputf.split(";")
self.InputFile = inputf
#FIXME: hardcode default detector model add check for detectorModel when submitting lcsim jobs!
#because currently no detectormodel is required!
if "IS_PROD" in self.workflow_commons:
if not self.detectorModel:
self.detectorModel = "clic_sid_cdr.zip"
if 'ExtraParams' in self.step_commons:
self.extraparams = self.step_commons['ExtraParams']
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"]:
#self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#if 'MokkaOutput' in self.workflow_commons:
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons[
'ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_rec_"):
self.outputREC = os.path.basename(obj)
elif obj.lower().count("_dst_"):
self.outputDST = os.path.basename(obj)
elif obj.lower().count("_sim_"):
self.InputFile = [os.path.basename(obj)]
else:
self.outputREC = getProdFilename(
self.outputREC,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.outputDST = getProdFilename(
self.outputDST,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if 'SLICOutput' in self.workflow_commons:
self.InputFile = [
getProdFilename(
self.workflow_commons["SLICOutput"],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
]
if 'aliasproperties' in self.step_commons:
self.aliasproperties = self.step_commons["aliasproperties"]
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".slcio") > -1:
self.InputFile.append(files)
self.log.info("Input files to treat %s" % self.InputFile)
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if self.WorkflowStartFrom:
self.startFrom = self.WorkflowStartFrom
if 'stdhepFile' in self.step_commons:
inputf = self.step_commons["stdhepFile"]
if not isinstance(inputf, list):
inputf = inputf.split(";")
self.InputFile = inputf
self.RandomSeed = self.determineRandomSeed()
resDBSlice = self.determineDBSlice()
if not resDBSlice['OK']:
return resDBSlice
if "IS_PROD" in self.workflow_commons:
if self.workflow_commons["IS_PROD"]:
self.mcRunNumber = self.RandomSeed
#self.OutputFile = getProdFilename(self.outputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons[
'ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_sim_"):
self.OutputFile = os.path.basename(obj)
elif obj.lower().count("_gen_"):
self.InputFile = [os.path.basename(obj)]
else:
self.OutputFile = getProdFilename(
self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]),
self.workflow_commons,
)
#if 'WhizardOutput' in self.workflow_commons:
# self.InputFile = getProdFilename(self.workflow_commons["WhizardOutput"],
# int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
self.InputFile = [
getProdFilename(
self.InputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
]
if len(self.InputData):
if 'EvtClass' in self.inputdataMeta and not self.ProcessID:
self.ProcessID = self.inputdataMeta['EvtClass']
if 'EvtType' in self.inputdataMeta and not self.ProcessID:
self.ProcessID = self.inputdataMeta['EvtType']
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".stdhep") > -1 or files.lower().find(
".hepevt") > -1:
self.InputFile.append(files)
break
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
@return: S_OK()
"""
if self.step_commons.has_key('inputXML'):
self.SteeringFile = self.step_commons['inputXML']
#TODO: Next is necessary for old interface, should be removed when old prods are archived.
if self.step_commons.has_key('outputREC'):
self.outputREC = self.step_commons['outputREC']
if self.step_commons.has_key('outputDST'):
self.outputDST = self.step_commons['outputDST']
if self.step_commons.has_key("inputSlcio"):
inputf = self.step_commons["inputSlcio"]
if not type(inputf) == types.ListType:
inputf = inputf.split(";")
self.InputFile = inputf
if self.step_commons.has_key('ExtraParams'):
self.extraparams = self.step_commons['ExtraParams']
if self.workflow_commons.has_key("IS_PROD"):
if self.workflow_commons["IS_PROD"]:
#self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#if self.workflow_commons.has_key("MokkaOutput"):
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if self.workflow_commons.has_key('ProductionOutputData'):
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_rec_"):
self.outputREC = os.path.basename(obj)
elif obj.lower().count("_dst_"):
self.outputDST = os.path.basename(obj)
elif obj.lower().count("_sim_"):
self.InputFile = [os.path.basename(obj)]
else:
self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if self.workflow_commons.has_key("SLICOutput"):
self.InputFile = [getProdFilename(self.workflow_commons["SLICOutput"],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))]
if self.step_commons.has_key("aliasproperties"):
self.aliasproperties = self.step_commons["aliasproperties"]
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".slcio") > -1:
self.InputFile.append(files)
self.log.info("Input files to treat %s" % self.InputFile)
return S_OK('Parameters resolved')
def constructProductionLFNs(paramDict):
""" Used for local testing of a workflow, a temporary measure until
LFN construction is tidied. This works using the workflow commons for
on the fly construction.
:param dict paramDict: dictionary with at least the keys ``PRODUCTION_ID``, ``JOB_ID``, ``outputList``
:returns: S_OK with ``ProductionOutputData``, ``LogFilePath``, ``LogTargetPath``
"""
result = checkForMandatoryKeys(paramDict,
['PRODUCTION_ID', 'JOB_ID', 'outputList'])
if not result['OK']:
return result
productionID = paramDict['PRODUCTION_ID']
jobID = paramDict['JOB_ID']
outputList = paramDict['outputList']
fileTupleList = []
gLogger.verbose('outputList %s' % (outputList))
for info in outputList:
#Nasty check on whether the created code parameters were not updated e.g. when changing defaults in a workflow
fileName = info['outputFile']
#rename to take care of correct path
fileName = getProdFilename(fileName, int(productionID), int(jobID))
fileTupleList.append((info['outputPath'], fileName))
#Get all LFN(s) to output data
outputData = []
for fileTuple in fileTupleList:
lfn = fileTuple[0] + "/" + str(productionID).zfill(8) + "/" + str(
int(jobID) / 1000).zfill(3) + "/" + fileTuple[1]
lfn = cleanUpLFNPath(lfn)
outputData.append(lfn)
#Get log file path - unique for all modules
## get First outputfile
basePath = fileTupleList[0][0]
#TODO adjust for ILD
logPath = basePath + "/" + str(productionID).zfill(8) + "/LOG"
#used for logFile upload to the LogSE
logFilePath = [
cleanUpLFNPath('%s/%s' % (logPath, str(int(jobID) / 1000).zfill(3)))
]
resLogs = getLogPath(dict(PRODUCTION_ID=productionID,
JOB_ID=jobID,
LogFilePath=logFilePath),
basePath=basePath)
if not resLogs['OK']:
return resLogs
logTargetPath = resLogs['Value']['LogTargetPath']
logFilePath = resLogs['Value']['LogFilePath']
if not outputData:
gLogger.info('No output data LFN(s) constructed')
else:
gLogger.verbose('Created the following output data LFN(s):\n%s' %
(string.join(outputData, '\n')))
gLogger.verbose('Log file path is:\n%s' % logFilePath[0])
gLogger.verbose('Log target path is:\n%s' % logTargetPath[0])
jobOutputs = {
'ProductionOutputData': outputData,
'LogFilePath': logFilePath,
'LogTargetPath': logTargetPath
}
return S_OK(jobOutputs)
def applicationSpecificInputs(self):
"""Resolve module input
:return: S_OK()
"""
self.parameters['ENERGY'] = self.energy
if not self.RandomSeed and self.jobID:
self.RandomSeed = self.jobID
if 'IS_PROD' in self.workflow_commons or 'IS_DBD_GEN_PROD' in self.workflow_commons:
self.RandomSeed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"])))
self.parameters['SEED'] = self.RandomSeed
self.parameters['NBEVTS'] = self.NumberOfEvents
self.parameters['LUMI'] = self.Lumi
##EVER USED???
if 'SusyModel' in self.step_commons:
self.susymodel = self.step_commons['SusyModel']
self.SteeringFile = os.path.basename(self.step_commons.get("InputFile", self.SteeringFile))
if self.SteeringFile == "whizard.in":
os.rename(self.SteeringFile, "whizardnew.in")
self.SteeringFile = "whizardnew.in"
self.parameters['PROCESS'] = self.evttype
listofparams = self.steeringparameters.split(";")
for param in listofparams:
if param.count("="):
self.parameters[param.split("=")[0]] = param.split("=")[1]
if self.OptionsDictStr:
LOG.info("Will use whizard.in definition from WhizardOptions.")
try:
self.optionsdict = eval(self.OptionsDictStr)
if 'integration_input' not in self.optionsdict:
self.optionsdict['integration_input'] = {}
if 'seed' not in self.optionsdict['integration_input']:
self.optionsdict['integration_input']['seed'] = int(self.RandomSeed)
if 'process_input' in self.optionsdict:
if 'sqrts' in self.optionsdict['process_input']:
self.energy = self.optionsdict['process_input']['sqrts']
except:
return S_ERROR("Could not convert string to dictionary for optionsdict")
if self.GenLevelCutDictStr:
LOG.info("Found generator level cuts")
try:
self.genlevelcuts = eval(self.GenLevelCutDictStr)
except:
return S_ERROR("Could not convert the generator level cuts back to dictionary")
if not len(self.SteeringFile) and not self.optionsdict:
self.getProcessInFile = True
if "IS_PROD" in self.workflow_commons:
if self.workflow_commons["IS_PROD"] and not self.willCut:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_gen_"):
self.OutputFile = os.path.basename(obj)
break
else:
#This is because most likely there is stdhepcut running after
self.OutputFile = "willcut.stdhep"
#getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if "IS_DBD_GEN_PROD" in self.workflow_commons and self.workflow_commons["IS_DBD_GEN_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
self.OutputFile = os.path.basename(obj)
break
else:
self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
return S_OK()
def applicationSpecificInputs(self):
""" By convention the module parameters are resolved here.
"""
self.log.verbose("Workflow commons:")
self.log.verbose(self.workflow_commons)
self.log.verbose("Step commons:")
self.log.verbose(self.step_commons)
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
if self.step_commons.has_key('TestFailover'):
self.enable = self.step_commons['TestFailover']
if not type(self.failoverTest) == type(True):
self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest)
self.failoverTest = False
if self.workflow_commons.has_key("PRODUCTION_ID"):
self.PRODUCTION_ID = self.workflow_commons["PRODUCTION_ID"]
if os.environ.has_key('JOBID'):
self.log.verbose('Found WMS JobID = %s' % self.jobID)
else:
self.log.info('No WMS JobID found, disabling module via control flag')
self.enable = False
if self.workflow_commons.has_key('Request'):
self.request = self.workflow_commons['Request']
else:
self.request = RequestContainer()
self.request.setRequestName('job_%s_request.xml' % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
##This is the thing that is used to establish the list of outpufiles to treat:
## Make sure that all that is in the : "listoutput" and also in the ProductionData
## is treated properly. Needed as whatever is in listoutput does not contain any reference to the
## prodID and task ID. Also if for some reason a step failed, then the corresponding data will not be there
if self.workflow_commons.has_key('outputList'):
self.outputList = self.workflow_commons['outputList']
if self.workflow_commons.has_key('ProductionOutputData'):
proddata = self.workflow_commons['ProductionOutputData'].split(";")
self.log.verbose("prod data : %s" % proddata )
olist = {}
for obj in self.outputList:
fname_in_outputlist = obj['outputFile'].lower()
extension = ''
if fname_in_outputlist.count("_sim") or fname_in_outputlist.count("_rec") or fname_in_outputlist.count("_dst"):
extension = ".slcio"
elif fname_in_outputlist.count("_gen"):
extension = ".stdhep"
fname_in_outputlist = fname_in_outputlist.replace(extension,"")
for prodfile in proddata:
prodfile = os.path.basename(prodfile)
extension = ''
if prodfile.count("_sim") or prodfile.count("_rec") or prodfile.count("_dst"):
extension = ".slcio"
elif prodfile.count("_gen"):
extension = ".stdhep"
prodfile = prodfile.replace(extension,"")
if olist.has_key(prodfile):
## This has already been treated, no need to come back to it.
continue
appdict = {}
if (fname_in_outputlist.count("_gen")):# and prodfile.lower().count("_gen_")) :
genf = obj['outputFile'].split("_gen")[0]
genf += "_gen"
if (prodfile.count(genf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
if (fname_in_outputlist.count("_sim")):
simf = obj['outputFile'].split("_sim")[0]
simf += "_sim"
if (prodfile.count(simf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
self.log.verbose('olist %s'%olist)
if (fname_in_outputlist.count("_rec")):
recf = obj['outputFile'].split("_rec")[0]
recf += "_rec"
if (prodfile.count(recf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
break
if (fname_in_outputlist.count("_dst") and prodfile.lower().count("_dst_")):
dstf = obj['outputFile'].split("_dst")[0]
dstf += "_dst"
if (prodfile.count(dstf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
break
self.outputList = olist.values()
else:
olist = []
for obj in self.outputList:
appdict = obj
appdict['outputFile'] = getProdFilename(obj['outputFile'],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
olist.append(appdict)
self.outputList = olist
self.log.verbose("OutputList : %s" % self.outputList)
if self.workflow_commons.has_key('outputMode'):
self.outputMode = self.workflow_commons['outputMode']
if self.workflow_commons.has_key('outputDataFileMask'):
self.outputDataFileMask = self.workflow_commons['outputDataFileMask']
if not type(self.outputDataFileMask) == type([]):
self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')]
#result = constructProductionLFNs(self.workflow_commons)
#if not result['OK']:
# self.log.error('Could not create production LFNs',result['Message'])
# return result
#self.prodOutputLFNs=result['Value']['ProductionOutputData']
if self.workflow_commons.has_key('ProductionOutputData'):
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" By convention the module parameters are resolved here.
"""
self.log.debug("Workflow commons: %s" % self.workflow_commons)
self.enable = self.step_commons.get('Enable', self.enable)
if not isinstance( self.enable, bool ):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
self.failoverTest = self.step_commons.get('TestFailover', self.failoverTest)
if not isinstance( self.failoverTest, bool ):
self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest)
self.failoverTest = False
self.productionID = self.workflow_commons.get("PRODUCTION_ID", self.productionID)
self.jobID = os.environ.get('JOBID', self.jobID)
if self.jobID:
self.log.verbose('Found WMS JobID = %s' % self.jobID)
else:
self.log.info('No WMS JobID found, disabling module via control flag')
self.enable = False
##This is the thing that is used to establish the list of outpufiles to treat:
## Make sure that all that is in the : "outputList" and also in the ProductionData
## is treated properly. Needed as whatever is in outputList does not contain any reference to the
## prodID and task ID. Also if for some reason a step failed, then the corresponding data will not be there
self.outputList = self.workflow_commons.get('outputList', self.outputList)
if self.outputList:
if 'ProductionOutputData' in self.workflow_commons:
productionData = self.workflow_commons['ProductionOutputData'].split(";")
self.log.verbose("prod data : %s" % productionData )
treatedOutputlist = {}
for expectedOutputfile in self.outputList:
self.log.debug("Treating file: %s" % expectedOutputfile['outputFile'])
self.getTreatedOutputlistNew(productionData, treatedOutputlist, expectedOutputfile)
self.outputList = treatedOutputlist.values()
else:
olist = []
for expectedOutputfile in self.outputList:
appdict = expectedOutputfile
appdict['outputFile'] = getProdFilename(expectedOutputfile['outputFile'],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
olist.append(appdict)
self.outputList = olist
self.log.verbose("OutputList : %s" % self.outputList)
self.outputMode = self.workflow_commons.get('outputMode', self.outputMode)
self.outputDataFileMask = self.workflow_commons.get('outputDataFileMask', self.outputDataFileMask)
if not isinstance( self.outputDataFileMask, list ):
self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')]
#result = constructProductionLFNs(self.workflow_commons)
#if not result['OK']:
# self.log.error('Could not create production LFNs',result['Message'])
# return result
#self.prodOutputLFNs=result['Value']['ProductionOutputData']
tempOutputLFNs = self.workflow_commons.get('ProductionOutputData', self.prodOutputLFNs)
if isinstance( tempOutputLFNs, basestring ):
self.prodOutputLFNs = tempOutputLFNs.split(";")
else:
self.prodOutputLFNs = tempOutputLFNs
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
@return: S_OK()
"""
if self.step_commons.has_key('numberOfEvents'):
self.NumberOfEvents = self.step_commons['numberOfEvents']
if self.step_commons.has_key('startFrom'):
self.startFrom = self.step_commons['startFrom']
#Need to keep until old prods are archived.
if self.step_commons.has_key("steeringFile"):
self.SteeringFile = self.step_commons['steeringFile']
if self.step_commons.has_key('stdhepFile'):
inputf = self.step_commons["stdhepFile"]
if not type(inputf) == types.ListType:
inputf = inputf.split(";")
self.InputFile = inputf
if self.step_commons.has_key('macFile'):
self.macFile = self.step_commons['macFile']
if self.step_commons.has_key('detectorModel'):
self.detectorModel = self.step_commons['detectorModel']
if self.step_commons.has_key('ProcessID'):
self.processID = self.step_commons['ProcessID']
if not self.RandomSeed:
if self.step_commons.has_key("RandomSeed"):
self.RandomSeed = self.step_commons["RandomSeed"]
elif self.jobID:
self.RandomSeed = self.jobID
if self.workflow_commons.has_key("IS_PROD"):
self.RandomSeed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"])))
if self.step_commons.has_key('dbSlice'):
self.dbSlice = self.step_commons['dbSlice']
if self.workflow_commons.has_key("IS_PROD"):
if self.workflow_commons["IS_PROD"]:
self.mcRunNumber = self.RandomSeed
#self.OutputFile = getProdFilename(self.outputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if self.workflow_commons.has_key('ProductionOutputData'):
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_sim_"):
self.OutputFile = os.path.basename(obj)
elif obj.lower().count("_gen_"):
self.InputFile = [os.path.basename(obj)]
else:
self.OutputFile = getProdFilename(self.OutputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
#if self.workflow_commons.has_key("WhizardOutput"):
# self.InputFile = getProdFilename(self.workflow_commons["WhizardOutput"],
# int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))]
if len(self.InputData):
if 'EvtClass' in self.inputdataMeta and not self.processID:
self.processID = self.inputdataMeta['EvtClass']
if 'EvtType' in self.inputdataMeta and not self.processID:
self.processID = self.inputdataMeta['EvtType']
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".stdhep") > -1 or files.lower().find(".hepevt") > -1:
self.InputFile.append(files)
break
return S_OK('Parameters resolved')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
@return: S_OK()
"""
##TODO: Need to keep for old interface. Move to ModuleBase
if self.step_commons.has_key('inputSlcio'):
inputf = self.step_commons["inputSlcio"]
if not type(inputf) == types.ListType:
if len(inputf):
inputf = inputf.split(";")
else:
inputf = []
self.InputFile = inputf
if self.workflow_commons.has_key('ParametricInputSandbox'):
paramsb = self.workflow_commons['ParametricInputSandbox']
if not type(paramsb) == types.ListType:
if len(paramsb):
paramsb = paramsb.split(";")
else:
paramsb = []
self.InputFile += paramsb
if self.step_commons.has_key('inputXML'):
self.SteeringFile = self.step_commons['inputXML']
if self.step_commons.has_key('inputGEAR'):
self.inputGEAR = self.step_commons['inputGEAR']
if self.step_commons.has_key('EvtsToProcess'):
self.NumberOfEvents = self.step_commons['EvtsToProcess']
##Backward compat needed, cannot remove yet.
if self.step_commons.has_key('outputREC'):
self.outputREC = self.step_commons['outputREC']
if self.step_commons.has_key('outputDST'):
self.outputDST = self.step_commons['outputDST']
if self.workflow_commons.has_key("IS_PROD"):
if self.workflow_commons["IS_PROD"] and len(self.OutputFile)==0:
#self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#if self.workflow_commons.has_key("MokkaOutput"):
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if self.workflow_commons.has_key('ProductionOutputData'):
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_rec_"):
self.outputREC = os.path.basename(obj)
elif obj.lower().count("_dst_"):
self.outputDST = os.path.basename(obj)
elif obj.lower().count("_sim_"):
self.InputFile = [os.path.basename(obj)]
else:
self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
#if self.workflow_commons.has_key("MokkaOutput"):
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))]
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".slcio") > -1:
self.InputFile.append(files)
return S_OK('Parameters resolved')
def test_getnames(self):
"""test ResolvePathsAndNames getNames..........................................................."""
res = getProdFilename("toto_gen.stdhep", 12345,123)
self.assertEqual(res,'toto_gen_12345_123.stdhep')
def applicationSpecificInputs(self):
"""Resolve module input
:return: S_OK()
"""
self.parameters['ENERGY'] = self.energy
if not self.RandomSeed and self.jobID:
self.RandomSeed = self.jobID
if 'IS_PROD' in self.workflow_commons or 'IS_DBD_GEN_PROD' in self.workflow_commons:
self.RandomSeed = int(
str(int(self.workflow_commons["PRODUCTION_ID"])) +
str(int(self.workflow_commons["JOB_ID"])))
self.parameters['SEED'] = self.RandomSeed
self.parameters['NBEVTS'] = self.NumberOfEvents
self.parameters['LUMI'] = self.Lumi
##EVER USED???
if 'SusyModel' in self.step_commons:
self.susymodel = self.step_commons['SusyModel']
self.SteeringFile = os.path.basename(
self.step_commons.get("InputFile", self.SteeringFile))
if self.SteeringFile == "whizard.in":
os.rename(self.SteeringFile, "whizardnew.in")
self.SteeringFile = "whizardnew.in"
self.parameters['PROCESS'] = self.evttype
listofparams = self.steeringparameters.split(";")
for param in listofparams:
if param.count("="):
self.parameters[param.split("=")[0]] = param.split("=")[1]
if self.OptionsDictStr:
self.log.info(
"Will use whizard.in definition from WhizardOptions.")
try:
self.optionsdict = eval(self.OptionsDictStr)
if 'integration_input' not in self.optionsdict:
self.optionsdict['integration_input'] = {}
if 'seed' not in self.optionsdict['integration_input']:
self.optionsdict['integration_input']['seed'] = int(
self.RandomSeed)
if 'process_input' in self.optionsdict:
if 'sqrts' in self.optionsdict['process_input']:
self.energy = self.optionsdict['process_input'][
'sqrts']
except:
return S_ERROR(
"Could not convert string to dictionary for optionsdict")
if self.GenLevelCutDictStr:
self.log.info("Found generator level cuts")
try:
self.genlevelcuts = eval(self.GenLevelCutDictStr)
except:
return S_ERROR(
"Could not convert the generator level cuts back to dictionary"
)
if not len(self.SteeringFile) and not self.optionsdict:
self.getProcessInFile = True
if "IS_PROD" in self.workflow_commons:
if self.workflow_commons["IS_PROD"] and not self.willCut:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons[
'ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_gen_"):
self.OutputFile = os.path.basename(obj)
break
else:
#This is because most likely there is stdhepcut running after
self.OutputFile = "willcut.stdhep"
#getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if "IS_DBD_GEN_PROD" in self.workflow_commons and self.workflow_commons[
"IS_DBD_GEN_PROD"]:
#self.OutputFile = getProdFilename(self.OutputFile,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons[
'ProductionOutputData'].split(";")
for obj in outputlist:
self.OutputFile = os.path.basename(obj)
break
else:
self.OutputFile = getProdFilename(
self.OutputFile,
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
return S_OK()
def constructProductionLFNs(paramDict):
""" Used for local testing of a workflow, a temporary measure until
LFN construction is tidied. This works using the workflow commons for
on the fly construction.
"""
keys = ['PRODUCTION_ID', 'JOB_ID', 'JobType', 'outputList']
for k in keys:
if not paramDict.has_key(k):
return S_ERROR('%s not defined' % k)
productionID = paramDict['PRODUCTION_ID']
jobID = paramDict['JOB_ID']
# wfMode = paramDict['dataType']
#wfLfnprefix=paramDict['lfnprefix']
#wfLfnpostfix=paramDict['lfnpostfix']
wfMask = ""
# wfMask = paramDict['outputDataFileMask']
if not type(wfMask) == type([]):
wfMask = [i.lower().strip() for i in wfMask.split(';')]
wfType = paramDict['JobType']
outputList = paramDict['outputList']
inputData = ''
if paramDict.has_key('InputData'):
inputData = paramDict['InputData']
res = getVOfromProxyGroup()
#res = gConfig.getOption("/DIRAC/VirtualOrganization", "ilc")
if not res['OK']:
gLogger.error('Could not get VO from CS, assuming ilc')
vo = 'ilc'
else:
vo = res['Value']
fileTupleList = []
#gLogger.verbose('wfLfnprefix = %s, wfLfnpostfix = %s, wfMask = %s, wfType=%s' %(wfLfnprefix,wfLfnpostfix,wfMask,wfType))
gLogger.verbose('outputList %s' % (outputList))
for info in outputList:
#Nasty check on whether the created code parameters were not updated e.g. when changing defaults in a workflow
fileName = info['outputFile']
#rename to take care of correct path
fileName = getProdFilename(fileName, int(productionID), int(jobID))
#index=0
#if not re.search('^\d',fileName[index]):
# index+=1
#if not fileName[index]==str(productionID).zfill(8):
# fileName[index]=str(productionID).zfill(8)
#if not fileName[index+1]==str(jobID).zfill(8):
# fileName[index+1]=str(jobID).zfill(8)
fileTupleList.append((info['outputPath'], fileName))
lfnRoot = ''
debugRoot = ''
#if inputData:
# gLogger.verbose('Making LFN_ROOT for job with inputdata: %s' %(inputData))
# lfnRoot = _getLFNRoot(inputData,wfLfnpostfix)
# debugRoot= _getLFNRoot('','debug',wfLfnpostfix)
#else:
# lfnRoot = _getLFNRoot('',wfLfnprefix,wfLfnpostfix)
# gLogger.verbose('LFN_ROOT is: %s' %(lfnRoot))
# debugRoot= _getLFNRoot('','debug',wfLfnpostfix)
#lfnRoot =
#gLogger.verbose('LFN_ROOT is: %s' %(lfnRoot))
#if not lfnRoot:
# return S_ERROR('LFN root could not be constructed')
#Get all LFN(s) to both output data and BK lists at this point (fine for BK)
outputData = []
#bkLFNs = []
debugLFNs = []
for fileTuple in fileTupleList:
#lfn = _makeProductionLfn(str(jobID).zfill(8),lfnRoot,fileTuple,wfLfnprefix,str(productionID).zfill(8))
lfn = fileTuple[0] + "/" + str(productionID).zfill(8) + "/" + str(int(jobID)/1000).zfill(3) + "/" + fileTuple[1]
if lfn.count('//'):
lfn = lfn.replace('//','/')
outputData.append(lfn)
#bkLFNs.append(lfn)
if debugRoot:
#debugLFNs.append(_makeProductionLfn(str(jobID).zfill(8),debugRoot,fileTuple,wfLfnprefix,str(productionID).zfill(8)))
debugLFNs.append("/" + vo + "/prod/debug/" + str(productionID).zfill(8))
#if debugRoot:
# debugLFNs.append(_makeProductionLfn(str(jobID).zfill(8),debugRoot,('%s_core' % str(jobID).zfill(8) ,'core'),wfLfnprefix,str(productionID).zfill(8)))
#Get log file path - unique for all modules
#logPath = _makeProductionPath(str(jobID).zfill(8),lfnRoot,'LOG',wfLfnprefix,str(productionID).zfill(8),log=True)
logPathtemp = fileTupleList[0][0].split("/")
logPathroot = string.join(logPathtemp[0:len(logPathtemp)-1], "/")
#TODO adjust for ILD
logPath = logPathroot + "/LOG/" + str(productionID).zfill(8)
logFilePath = ['%s/%s' % (logPath, str(int(jobID)/1000).zfill(3))]
logTargetPath = ['%s/%s_%s.tar' % (logPath, str(productionID).zfill(8), str(int(jobID)).zfill(3))]
#[ aside, why does makeProductionPath not append the jobID itself ????
# this is really only used in one place since the logTargetPath is just written to a text file (should be reviewed)... ]
#Strip output data according to file mask
if wfMask:
newOutputData = []
#newBKLFNs = []
for od in outputData:
for i in wfMask:
if re.search('.%s$' % i, od):
if not od in newOutputData:
newOutputData.append(od)
#for bk in bkLFNs:
# newBKLFNs.append(bk)
outputData = newOutputData
#bkLFNs = newBKLFNs
if not outputData:
gLogger.info('No output data LFN(s) constructed')
else:
gLogger.verbose('Created the following output data LFN(s):\n%s' % (string.join(outputData,'\n')))
gLogger.verbose('Log file path is:\n%s' % logFilePath[0])
gLogger.verbose('Log target path is:\n%s' % logTargetPath[0])
#if bkLFNs:
# gLogger.verbose('BookkeepingLFN(s) are:\n%s' %(string.join(bkLFNs,'\n')))
if debugLFNs:
gLogger.verbose('DebugLFN(s) are:\n%s' % (string.join(debugLFNs, '\n')))
jobOutputs = {'ProductionOutputData' : outputData, 'LogFilePath' : logFilePath,
'LogTargetPath' : logTargetPath, 'DebugLFNs' : debugLFNs}
return S_OK(jobOutputs)
def test_getnames(self):
"""test ResolvePathsAndNames getNames..........................................................."""
res = getProdFilename("toto_gen.stdhep", 12345, 123)
self.assertEqual(res, 'toto_gen_12345_123.stdhep')
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if 'inputXML' in self.step_commons:
self.SteeringFile = self.step_commons['inputXML']
#TODO: Next is necessary for old interface, should be removed when old prods are archived.
if 'outputREC' in self.step_commons:
self.outputREC = self.step_commons['outputREC']
if 'outputDST' in self.step_commons:
self.outputDST = self.step_commons['outputDST']
if 'inputSlcio' in self.step_commons:
inputf = self.step_commons["inputSlcio"]
if not isinstance( inputf, list ):
inputf = inputf.split(";")
self.InputFile = inputf
#FIXME: hardcode default detector model add check for detectorModel when submitting lcsim jobs!
#because currently no detectormodel is required!
if "IS_PROD" in self.workflow_commons:
if not self.detectorModel:
self.detectorModel = "clic_sid_cdr.zip"
if 'ExtraParams' in self.step_commons:
self.extraparams = self.step_commons['ExtraParams']
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"]:
#self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#if 'MokkaOutput' in self.workflow_commons:
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_rec_"):
self.outputREC = os.path.basename(obj)
elif obj.lower().count("_dst_"):
self.outputDST = os.path.basename(obj)
elif obj.lower().count("_sim_"):
self.InputFile = [os.path.basename(obj)]
else:
self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
if 'SLICOutput' in self.workflow_commons:
self.InputFile = [getProdFilename(self.workflow_commons["SLICOutput"],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))]
if 'aliasproperties' in self.step_commons:
self.aliasproperties = self.step_commons["aliasproperties"]
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
if files.lower().find(".slcio") > -1:
self.InputFile.append(files)
LOG.info("Input files to treat %s" % self.InputFile)
return S_OK('Parameters resolved')