def CommonSpecialFinalConfig(self):
#These are semi-hacks... please minimize the stuff here!
if self.inDic.has_key('skipEvents'):
skip = self.inDic['skipEvents']
print "INFO skipEvents is special. It is only added to the first sub-step(s)."
firstSteps = self.GetFirstSubStep()
if firstSteps == None:
raise RuntimeError(
"skipEvents requested, but can't find which sub-step it should be added to."
)
for step in firstSteps:
self.AddToDic(self.ValidSubStepsDic[step], 'skipEvents', skip,
"create")
print "skipEvents=%i added to %s." % (int(skip), step)
#Remove duplicate outputs, giving precedence to the first running step of the substeps list from ConfigDic
from PATJobTransforms.Configuration import ConfigDic
for argKey in ConfigDic.keys():
argObj = ConfigDic[argKey]
if hasattr(argObj, "isOutput") and hasattr(argObj, "subSteps"):
steps = argObj.subSteps
alreadyProduced = False
for step in steps:
if self.SubStepIsExecuted(step) and self.ValidSubStepsDic[
step].has_key(argKey):
if alreadyProduced:
self.ValidSubStepsDic[step].pop(argKey)
else:
alreadyProduced = True
return
def CommonSpecialFinalConfig(self):
#These are semi-hacks... please minimize the stuff here!
if self.inDic.has_key('skipEvents'):
skip=self.inDic['skipEvents']
print "INFO skipEvents is special. It is only added to the first sub-step(s)."
firstSteps=self.GetFirstSubStep()
if firstSteps==None:
raise RuntimeError("skipEvents requested, but can't find which sub-step it should be added to.")
for step in firstSteps:
self.AddToDic(self.ValidSubStepsDic[step],'skipEvents',skip,"create")
print "skipEvents=%i added to %s."%(int(skip),step)
#Remove duplicate outputs, giving precedence to the first running step of the substeps list from ConfigDic
from PATJobTransforms.Configuration import ConfigDic
for argKey in ConfigDic.keys():
argObj=ConfigDic[argKey]
if hasattr(argObj,"isOutput") and hasattr(argObj,"subSteps"):
steps=argObj.subSteps
alreadyProduced=False
for step in steps:
if self.SubStepIsExecuted(step) and self.ValidSubStepsDic[step].has_key(argKey):
if alreadyProduced:
self.ValidSubStepsDic[step].pop(argKey)
else:
alreadyProduced=True
return
def addTrfOutput(self,key):
from PATJobTransforms.Configuration import AddUserDPD,ConfigDic
if ConfigDic.has_key(key):
raise RuntimeError("key %s already exists, cannot be redefined"%key)
tmpClass=TmpClass(key)
AddUserDPD(tmpClass, ['e2d','a2d'])
self.addOutput(key,'defaultFileName.root')
return
def addTrfOutput(self, key):
from PATJobTransforms.Configuration import AddUserDPD, ConfigDic
if ConfigDic.has_key(key):
raise RuntimeError("key %s already exists, cannot be redefined" %
key)
tmpClass = TmpClass(key)
AddUserDPD(tmpClass, ['e2d', 'a2d'])
self.addOutput(key, 'defaultFileName.root')
return
def hasOutput(self,dic):
from PATJobTransforms.Configuration import ConfigDic
for key in dic.keys():
if ConfigDic.has_key(key):
if hasattr(ConfigDic[key],"isOutput"):
return True
else:
pass
else:
raise RuntimeError("key %s is not defined in ConfigDic"%key)
pass
return False
def hasInput(self, dic):
from PATJobTransforms.Configuration import ConfigDic
for key in dic.keys():
if ConfigDic.has_key(key):
if hasattr(ConfigDic[key], "isInput"):
return True
else:
pass
else:
raise RuntimeError("key %s is not defined in ConfigDic" % key)
pass
return False
def CheckTreeName(self, inDic):
from PyJobTransformsCore.full_trfarg import InputNtupleFileArg
# Check tree name from input file
for key in inDic.keys():
if key.startswith('inputNTUP') and key.endswith(
'File') and ConfigDic.has_key(key):
trf = set()
ConfigDic[key](trf=trf, inDic={})
for x in trf:
if isinstance(x, InputNtupleFileArg):
self.tree_name = x._fileType.tree_names[0]
return
def AutoConfigureFromDic(trf,zeDic):
#configure the transform from zeDic, but also modify zeDic as needed
origDic=zeDic.copy()
#loop over origDic, since zeDic may be modified by the functions returned by ConfigDic
for key in origDic.keys():
if ConfigDic.has_key(key):
ConfigDic[key](trf,zeDic)
elif KeyExceptionCatcher(key) is "OK":
print "No ConfigDic entry for key '%s'."
print "This allowed for special options like: --ignoreunknown, --athenaopts, -h, etc."
else:
raise RuntimeError("key %s is not defined in ConfigDic"%key)
return zeDic
def getPotentialArgs(self):
from PyJobTransformsCore.trfutil import AddValidItemToList,OverlapLists
trfSteps=self.ValidSubStepsDic.keys()
AddValidItemToList(BasicCommonSteps,trfSteps)
trfArgs=[]
from PATJobTransforms.Configuration import ConfigDic
for key in ConfigDic.keys():
if hasattr(ConfigDic[key],"subSteps"):
keySteps=ConfigDic[key].subSteps
if OverlapLists(trfSteps,keySteps):
trfArgs.append(key)
pass
pass
else:
trfArgs.append(key)
pass
pass
return trfArgs
def getPotentialArgs(self):
from PyJobTransformsCore.trfutil import AddValidItemToList, OverlapLists
trfSteps = self.ValidSubStepsDic.keys()
AddValidItemToList(BasicCommonSteps, trfSteps)
trfArgs = []
from PATJobTransforms.Configuration import ConfigDic
for key in ConfigDic.keys():
if hasattr(ConfigDic[key], "subSteps"):
keySteps = ConfigDic[key].subSteps
if OverlapLists(trfSteps, keySteps):
trfArgs.append(key)
pass
pass
else:
trfArgs.append(key)
pass
pass
return trfArgs
def ConfigureCommonInternalSubSteps(self):
from PATJobTransforms.Configuration import ConfigDic
#Loop over all inputDic keys
for key in self.inDic.keys():
newKey, subStep = self.GetKeyAndSubStep(key)
if not ConfigDic.has_key(newKey):
raise RuntimeError("key '%s' is not defined in ConfigDic" %
newKey)
#Set subSteps from above if specified, otherwise use default value from ConfigDic
subSteps = [subStep]
if subStep == None:
subSteps = []
tmpSteps = ConfigDic[newKey].subSteps
for t in tmpSteps:
if BasicCommonSteps.__contains__(
t) or self.ValidSubStepsDic.keys().__contains__(t):
subSteps.append(t)
meth = "update"
#meth="create" <--- problematic, leave it off (OK)
#Now configure the sub-step(s) for this key
for step in subSteps:
if step == "any":
self.AddToAllDics(newKey, self.inDic[key], meth)
elif step == "none":
#these are typically for Reco_trf only configuration (e.g. DBRelease)
ConfigDic[newKey](self, self.inDic)
elif step == "first":
#handle this later, when the input/output configuration is known.
pass
elif self.ValidSubStepsDic.has_key(step):
self.AddToDic(self.ValidSubStepsDic[step], newKey,
self.inDic[key], meth)
else:
raise RuntimeError(
"Unexpected sub-step '%s' for key '%s'" %
(step, newKey))
return
def WriteRecoTrfSignature():
signatureDic={}
signatureDic['InputAndOutputFileTypes']={}
signatureDic['phconfig']=[]
#A few examples of inputs and their possible outputs... (not all yet)
signatureDic['InputAndOutputFileTypes']['inputESDFile']=['outputAODFile','outputESDFile','outputNTUP_TRIGFile','outputDESDM_EGAMMAFile']
signatureDic['InputAndOutputFileTypes']['inputBSFile']=['outputBSFile','outputESDFile','outputCBNTFile','outputNTUP_TRKVALIDFile','outputTAG_COMMFile']
signatureDic['InputAndOutputFileTypes']['inputAODFile']=['outputTAGFile','outputDAODM_SGLMUFile']
#A few examples of physics configuration parameters... (not all yet)
signatureDic['phconfig'].append('autoConfiguration')
signatureDic['phconfig'].append('preExec')
signatureDic['phconfig'].append('postExec')
signatureDic['phconfig'].append('preInclude')
signatureDic['phconfig'].append('postInclude')
signatureDic['phconfig'].append('geometryVersion')
signatureDic['phconfig'].append('conditionsTag')
signatureDic['phconfig'].append('beamType')
outName="Reco_trf.signature.pickle"
f = open(outName, 'w')
pickle.dump(signatureDic, f)
f.close()
print "salut Pavel"
#Towards an automated writing of the pickled dic...
from PATJobTransforms.Configuration import ConfigDic
#searching for outputs from ESD input
for key in ConfigDic.keys():
obj=ConfigDic[key]
if hasattr(obj,'isOutput') and hasattr(obj,'subSteps'):
for s in obj.subSteps:
if s.startswith('e2'):
print key
def ConfigureCommonInternalSubSteps(self):
from PATJobTransforms.Configuration import ConfigDic
#Loop over all inputDic keys
for key in self.inDic.keys():
newKey,subStep=self.GetKeyAndSubStep(key)
if not ConfigDic.has_key(newKey):
raise RuntimeError("key '%s' is not defined in ConfigDic"%newKey)
#Set subSteps from above if specified, otherwise use default value from ConfigDic
subSteps=[subStep]
if subStep==None:
subSteps=[]
tmpSteps=ConfigDic[newKey].subSteps
for t in tmpSteps:
if BasicCommonSteps.__contains__(t) or self.ValidSubStepsDic.keys().__contains__(t):
subSteps.append(t)
meth="update"
#meth="create" <--- problematic, leave it off (OK)
#Now configure the sub-step(s) for this key
for step in subSteps:
if step=="any":
self.AddToAllDics(newKey,self.inDic[key],meth)
elif step=="none":
#these are typically for Reco_trf only configuration (e.g. DBRelease)
ConfigDic[newKey](self,self.inDic)
elif step=="first":
#handle this later, when the input/output configuration is known.
pass
elif self.ValidSubStepsDic.has_key(step):
self.AddToDic(self.ValidSubStepsDic[step],newKey,self.inDic[key],meth)
else:
raise RuntimeError("Unexpected sub-step '%s' for key '%s'"%(step,newKey))
return
else: currentConditionsTag += resolveAlias.getCurrentES()
# Starting in 2012 all magnetic field configs are in one tag. Don't put this in the autoconfiguration part.
if '*' not in currentConditionsTag:
currentConditionsTag = currentConditionsTag[14:]
else:
# ConditionsTag in autoConfiguration is not changed.
updateConditionsTag = False
#prepare input, output and config dictionaries
inputDic={}
outputDic={}
configDic={}
from PATJobTransforms.Configuration import ConfigDic
didConditionsUpdate=False
for key in startingDic.keys():
if ConfigDic.has_key(key) and hasattr(ConfigDic[key],"isInput"):
inputDic[key]={}
elif ConfigDic.has_key(key) and hasattr(ConfigDic[key],"isOutput") and (key!="tmpESD" and key!="tmpAOD"):
try:
#print "key: " , key , " isOutput"
outputDic[key]=OutputsVsStreams[key]
except:
print "Known outputs defined in the OutputsVsStreams dictionnary are:"
print OutputsVsStreams
raise RuntimeError("Don't know what to do with out key %s, please add it to OutputsVsStreams to use it"%key)
elif key=='autoConfiguration' and updateConditionsTag and 'Conditions' in currentConditionsTag: # legacy: used for autoconfiguration of conditions like COMCOND-ES1P*-005-04
if(startingDic[key].find(currentConditionsTag)>=0):
print '*** autoConfiguration: %s (CURRENT) is already specified ***' % (currentConditionsTag)
elif(startingDic[key].find('ConditionsTag')>=0): # ConditionsTag is already defined.
autoConfs = startingDic[key].split(',')
for i in range(len(autoConfs)):
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
__doc__ = """NTUP -> NTUP skimming/thinning files."""
ListOfDefaultPositionalKeys = [
'maxEvents', 'skipEvents', 'preInclude', 'postInclude', 'preExec',
'postExec', 'topOptions', 'AMITag', 'tree_name', '--ignoreerrors',
'--athenaopts', '--omitvalidation'
]
from PATJobTransforms.Configuration import ConfigDic
for key in ConfigDic.keys():
if key.startswith('inputNTUP') and key.endswith('File'):
ListOfDefaultPositionalKeys.append(key)
from NTUPtoNTUPCore.NTUPtoNTUPProdFlags import listAllKnownNTUPtoNTUP
for ntup in listAllKnownNTUPtoNTUP:
dInName = 'input' + (ntup.StreamName.lstrip("Stream")) + 'File'
ListOfDefaultPositionalKeys.append(dInName)
dOutName = 'output' + (ntup.StreamName.lstrip("Stream")) + 'File'
ListOfDefaultPositionalKeys.append(dOutName)
from PATJobTransforms.BaseOfBasicTrf import BaseOfBasicTrf
from PyJobTransformsCore.trf import Author
from PyJobTransformsCore.trf import JobTransform
class NTUPtoNTUPJobTransform(BaseOfBasicTrf):
def __init__(self, inDic):
JobTransform.__init__(
self,
# Skeleton top job options for MergeNTUP_trf
#
#==============================================================
import subprocess
from PATJobTransforms.Configuration import ConfigDic
from PyJobTransformsCore.trferr import TransformArgumentError
theApp.EvtMax = 0
inFileArgs = 0
outFileArgs = 0
inKey = None
outKey = None
for key in ConfigDic.iterkeys():
if key.startswith('outputNTUP') and key.endswith('File') and hasattr(
runArgs, key):
outFileArgs += 1
outKey = key
outFile = getattr(runArgs, key)
print "Using argument ", key, " = ", outFile
if key.startswith('inputNTUP') and key.endswith('File') and hasattr(
runArgs, key):
inFileArgs += 1
inKey = key
inFile = getattr(runArgs, key)
print "Using argument ", key, " = ", inFile
if outFileArgs != 1:
raise TransformArgumentError(
# Skeleton top job options for MergeNTUP_trf
#
#==============================================================
import subprocess
from PATJobTransforms.Configuration import ConfigDic
from PyJobTransformsCore.trferr import TransformArgumentError
theApp.EvtMax = 0
inFileArgs=0
outFileArgs=0
inKey=None
outKey=None
for key in ConfigDic.iterkeys():
if key.startswith('outputNTUP') and key.endswith('File') and hasattr(runArgs,key):
outFileArgs+=1
outKey=key
outFile=getattr(runArgs,key)
print "Using argument ", key, " = ",outFile
if key.startswith('inputNTUP') and key.endswith('File') and hasattr(runArgs,key):
inFileArgs+=1
inKey=key
inFile=getattr(runArgs,key)
print "Using argument ", key, " = ",inFile
if outFileArgs!=1:
raise TransformArgumentError(message='Wrong number of outputNTUPXXXFile arguments: {0:d} instead of 1. Stopping!'.format(outFileArgs))
if inFileArgs!=1:
def DicInputs(aDic):
from PATJobTransforms.Configuration import ConfigDic
for key in aDic.keys():
if ConfigDic.has_key(key) and hasattr(ConfigDic[key],"isInput"):
return aDic[key]
return ""
def DicHasOutputs(aDic):
from PATJobTransforms.Configuration import ConfigDic
for key in aDic.keys():
if ConfigDic.has_key(key) and hasattr(ConfigDic[key],"isOutput"):
return True
return False
def UpdateDicWithAMI(userDic,amiTag,info):
amiInputDic=info['amiInputDic']
amiOuputDic=info['amiOuputDic']
amiPhysDic=info['amiPhysDic']
PopSynonyms(amiInputDic)
PopSynonyms(amiOuputDic)
PopSynonyms(amiPhysDic)
#Now update userDic, becoming outDic
outDic={}
#The rules are:
# 1) if userDic specifies an input, use it. Otherwise use the default
# 2) if userDic specifies one output, overwrite whole amiOuputDic. Otherwise use all amiOuputDic with default values.
# 3) any physConfig specified in userDic overwrites amiConfigDic, unless the append option is used.
#inputs
inputFileValue=DicInputs(userDic)
if inputFileValue=="":
print "\n"
if len(amiInputDic.keys())>0:
from PATJobTransforms.DefaultInputs import DefaultInputs
#inKey=amiInputDic.keys()[0]
for inKey in amiInputDic.keys():
if DefaultInputs.has_key(inKey):
inputFileValue=DefaultInputs[inKey]
if amiTag=="q120": inputFileValue=DefaultInputs["cosmicsBS"]
elif amiTag=="q126": inputFileValue=DefaultInputs["topBS"]
outDic[inKey]=inputFileValue
print "INFO Using default input value: %s=%s"%(inKey,outDic[inKey])
else:
raise RuntimeError("Key %s is not defined in DefaultInputs"%inKey)
#outputs
#if no output is specified, use default values for all those specified in AMI tag if input matches regular expression
if DicHasOutputs(userDic):
print "\nUsing outputs specified by user, bypassing those from AMI."
else:
print "\n"
from PATJobTransforms.Configuration import ConfigDic
from PATJobTransforms.TrfFlags import trfFlags
import re
for key in amiOuputDic.keys():
if not ConfigDic.has_key(key):
raise RuntimeError("Key %s from amiOutputDic is not known by job transform ConfigDic"%key)
if not hasattr(ConfigDic[key],"isOutput"):
raise RuntimeError("Key %s from amiOutputDic is not known as an output by job transform ConfigDic"%key)
pattern=".*" #trivial pattern that always matches
if amiOuputDic[key].has_key('ifMatch'):
pattern=amiOuputDic[key]['ifMatch'] #overwrites trivial pattern with the one from AMI
idx=inputFileValue.rfind("/")
if idx != -1:
inputFileValue=inputFileValue[1+idx:]
print "Reduced input file Name:",inputFileValue
if (not trfFlags.ApplyIfMatchPattern()) or re.match(pattern,inputFileValue):
type=ConfigDic[key].isOutput
defaultValue=None
if type=='bs':
defaultValue='my'+str(amiOuputDic[key]['dstype'])+'.data'
elif type=='root':
defaultValue='my'+str(amiOuputDic[key]['dstype'])+'.root'
elif type=='pool':
defaultValue='my'+str(amiOuputDic[key]['dstype'])+'.pool.root'
else:
raise RuntimeError("Don't know to define a default value for type %s"%type)
outDic[key]=defaultValue
print "INFO Using default output value: %s=%s"%(key,outDic[key])
else:
print "INFO %s not produced since input file '%s' does not match pattern '%s'."%(key,inputFileValue,pattern)
#physics_configuration: take it all
outDic.update(amiPhysDic)
#now update outDic with everything from userDic (including input/output if relevant)
#first look for append option...
AppendDic1WithDic2(outDic,userDic)
#print "OutDict after append",outDic
#at this point userDic will supercede what was in AMI in case of conflicts
outDic.update(userDic)
#Call PopSynonyms_DRAWOutput here and in BuildDicFromCommandLine to be sure it's executed in any case
PopSynonyms_DRAWOutput(outDic)
return outDic,info