def fixSubsetCol( var, nSorts = 1, nEta = 1, nEt = 1, level = None ):
"""
Helper method to correct variable to be a looping bound collection
correctly represented by a LoopingBoundsCollection instance.
"""
tree_types = (SubsetGeneratorCollection, SubsetGeneratorPatterns, list, tuple )
try:
# Retrieve collection maximum depth
_, _, _, _, depth = traverse(var, tree_types = tree_types).next()
except GeneratorExit:
depth = 0
if depth < 5:
if depth == 0:
var = [[[[var]]]]
elif depth == 1:
var = [[[var]]]
elif depth == 2:
var = [[var]]
elif depth == 3:
var = [var]
# We also want to be sure that they are in correct type and correct size:
from RingerCore import inspect_list_attrs
var = inspect_list_attrs(var, 3, SubsetGeneratorPatterns , tree_types = tree_types, level = level )
var = inspect_list_attrs(var, 2, SubsetGeneratorCollection, tree_types = tree_types, dim = nSorts, name = "nSorts", )
var = inspect_list_attrs(var, 1, SubsetGeneratorCollection, tree_types = tree_types, dim = nEta, name = "nEta", )
var = inspect_list_attrs(var, 0, SubsetGeneratorCollection, tree_types = tree_types, dim = nEt, name = "nEt", deepcopy = True )
else:
raise ValueError("subset generator dimensions size is larger than 5.")
return var
def select(fl, filters, popListInCaseOneItem=True):
"""
Return a selection from fl maching f
WARNING: This selection method retrieves the same string contained in fl
if it matches two different filters.
"""
try:
iter(filters)
if isinstance(filters, basestring): raise Exception
except:
filters = [filters]
ret = []
from RingerCore import traverse
for filt in filters:
taken = filter(lambda obj: type(obj) in (str, unicode) and filt in obj,
traverse(fl, simple_ret=True))
ret.append(taken)
if popListInCaseOneItem and len(ret) == 1: ret = ret[0]
return ret
[0.9892, 0.9869, 0.9431, 0.9834, 0.9633],
[0.9914, 0.9904, 0.9724, 0.9885, 0.9680],
[0.9930, 0.9921, 0.9778, 0.9763, 0.9533],
[0.9938, 0.9933, 0.9794, 0.9925, 0.9826]]) * 100.
#etaBins = [0, 0.8]
#for ref in (veryloose20160701, loose20160701, medium20160701, tight20160701):
ref = tight20160701
from RingerCore import traverse
pdrefs = ref
#print pdrefs
pfrefs = np.array([[0.05] * len(etaBins)] * len(etBins)) * 100. # 3 5 7 10
efficiencyValues = np.array([
np.array([refs])
for refs in zip(traverse(pdrefs, tree_types=(np.ndarray), simple_ret=True),
traverse(pfrefs, tree_types=(np.ndarray), simple_ret=True))
]).reshape(pdrefs.shape + (2, ))
basePath = '/eos/user/j/jodafons/CERN-DATA/data/data17_13TeV/'
sgnInputFile = 'EGAM1'
bkgInputFile = 'EGAM7'
outputFile = 'sample'
treePath = ["*/HLT/Physval/Egamma/probes", "*/HLT/Physval/Egamma/fakes"]
import os.path
from TuningTools import Reference, RingerOperation, Detector
from TuningTools import createData
from RingerCore import LoggingLevel
from TuningTools.dataframe import Dataframe
from RingerCore.Configure import Development
def __call__( self, fList, ringerOperation, **kw):
"""
Read ntuple and return patterns and efficiencies.
Arguments:
- fList: The file path or file list path. It can be an argument list of
two types:
o List: each element is a string path to the file;
o Comma separated string: each path is separated via a comma
o Folders: Expand folders recursively adding also files within them to analysis
- ringerOperation: Set Operation type. It can be both a string or the
RingerOperation
Optional arguments:
- filterType [None]: whether to filter. Use FilterType enumeration
- reference [Truth]: set reference for targets. Use Reference enumeration
- treePath [Set using operation]: set tree name on file, this may be set to
use different sources then the default.
Default for:
o Offline: Offline/Egamma/Ntuple/electron
o L2: Trigger/HLT/Egamma/TPNtuple/e24_medium_L1EM18VH
- l1EmClusCut [None]: Set L1 cluster energy cut if operating on the trigger
- l2EtCut [None]: Set L2 cluster energy cut value if operating on the trigger
- offEtCut [None]: Set Offline cluster energy cut value
- nClusters [None]: Read up to nClusters. Use None to run for all clusters.
- getRatesOnly [False]: Read up to nClusters. Use None to run for all clusters.
- etBins [None]: E_T bins (GeV) where the data should be segmented
- etaBins [None]: eta bins where the data should be segmented
- ringConfig [100]: A list containing the number of rings available in the data
for each eta bin.
- crossVal [None]: Whether to measure benchmark efficiency splitting it
by the crossVal-validation datasets
- extractDet [None]: Which detector to export (use Detector enumeration).
Defaults are:
o L2Calo: Calorimetry
o L2: Tracking
o Offline: Calorimetry
o Others: CaloAndTrack
- standardCaloVariables [False]: Whether to extract standard track variables.
- useTRT [False]: Whether to export TRT information when dumping track
variables.
- supportTriggers [True]: Whether reading data comes from support triggers
"""
# Offline information branches:
__offlineBranches = ['el_et',
'el_eta',
#'el_loose',
#'el_medium',
#'el_tight',
'el_lhLoose',
'el_lhMedium',
'el_lhTight',
'mc_hasMC',
'mc_isElectron',
'mc_hasZMother',
'el_nPileupPrimaryVtx',
]
# Online information branches
__onlineBranches = []
__l2stdCaloBranches = ['trig_L2_calo_et',
'trig_L2_calo_eta',
'trig_L2_calo_phi',
'trig_L2_calo_e237', # rEta
'trig_L2_calo_e277', # rEta
'trig_L2_calo_fracs1', # F1: fraction sample 1
'trig_L2_calo_weta2', # weta2
'trig_L2_calo_ehad1', # energy on hadronic sample 1
'trig_L2_calo_emaxs1', # eratio
'trig_L2_calo_e2tsts1', # eratio
'trig_L2_calo_wstot',] # wstot
__l2trackBranches = [ # Do not add non patter variables on this branch list
#'trig_L2_el_pt',
#'trig_L2_el_eta',
#'trig_L2_el_phi',
#'trig_L2_el_caloEta',
#'trig_L2_el_charge',
#'trig_L2_el_nTRTHits',
#'trig_L2_el_nTRTHiThresholdHits',
'trig_L2_el_etOverPt',
'trig_L2_el_trkClusDeta',
'trig_L2_el_trkClusDphi',]
# Retrieve information from keyword arguments
filterType = retrieve_kw(kw, 'filterType', FilterType.DoNotFilter )
reference = retrieve_kw(kw, 'reference', Reference.Truth )
l1EmClusCut = retrieve_kw(kw, 'l1EmClusCut', None )
l2EtCut = retrieve_kw(kw, 'l2EtCut', None )
efEtCut = retrieve_kw(kw, 'efEtCut', None )
offEtCut = retrieve_kw(kw, 'offEtCut', None )
treePath = retrieve_kw(kw, 'treePath', None )
nClusters = retrieve_kw(kw, 'nClusters', None )
getRates = retrieve_kw(kw, 'getRates', True )
getRatesOnly = retrieve_kw(kw, 'getRatesOnly', False )
etBins = retrieve_kw(kw, 'etBins', None )
etaBins = retrieve_kw(kw, 'etaBins', None )
crossVal = retrieve_kw(kw, 'crossVal', None )
ringConfig = retrieve_kw(kw, 'ringConfig', 100 )
extractDet = retrieve_kw(kw, 'extractDet', None )
standardCaloVariables = retrieve_kw(kw, 'standardCaloVariables', False )
useTRT = retrieve_kw(kw, 'useTRT', False )
supportTriggers = retrieve_kw(kw, 'supportTriggers', True )
monitoring = retrieve_kw(kw, 'monitoring', None )
pileupRef = retrieve_kw(kw, 'pileupRef', NotSet )
import ROOT, pkgutil
#gROOT.ProcessLine (".x $ROOTCOREDIR/scripts/load_packages.C");
#ROOT.gROOT.Macro('$ROOTCOREDIR/scripts/load_packages.C')
if not( bool( pkgutil.find_loader( 'libTuningTools' ) ) and ROOT.gSystem.Load('libTuningTools') >= 0 ) and \
not( bool( pkgutil.find_loader( 'libTuningToolsLib' ) ) and ROOT.gSystem.Load('libTuningToolsLib') >= 0 ):
#ROOT.gSystem.Load('libTuningToolsPythonLib') < 0:
self._fatal("Could not load TuningTools library", ImportError)
if 'level' in kw: self.level = kw.pop('level')
# and delete it to avoid mistakes:
checkForUnusedVars( kw, self._warning )
del kw
### Parse arguments
# Mutual exclusive arguments:
if not getRates and getRatesOnly:
self._logger.error("Cannot run with getRates set to False and getRatesOnly set to True. Setting getRates to True.")
getRates = True
# Also parse operation, check if its type is string and if we can
# transform it to the known operation enum:
fList = csvStr2List ( fList )
fList = expandFolders( fList )
ringerOperation = RingerOperation.retrieve(ringerOperation)
reference = Reference.retrieve(reference)
if isinstance(l1EmClusCut, str):
l1EmClusCut = float(l1EmClusCut)
if l1EmClusCut:
l1EmClusCut = 1000.*l1EmClusCut # Put energy in MeV
__onlineBranches.append( 'trig_L1_emClus' )
if l2EtCut:
l2EtCut = 1000.*l2EtCut # Put energy in MeV
__onlineBranches.append( 'trig_L2_calo_et' )
if efEtCut:
efEtCut = 1000.*efEtCut # Put energy in MeV
__onlineBranches.append( 'trig_EF_calo_et' )
if offEtCut:
offEtCut = 1000.*offEtCut # Put energy in MeV
__offlineBranches.append( 'el_et' )
if not supportTriggers:
__onlineBranches.append( 'trig_L1_accept' )
# Check if treePath is None and try to set it automatically
if treePath is None:
treePath = 'Offline/Egamma/Ntuple/electron' if ringerOperation < 0 else \
'Trigger/HLT/Egamma/TPNtuple/e24_medium_L1EM18VH'
# Check whether using bins
useBins=False; useEtBins=False; useEtaBins=False
nEtaBins = 1; nEtBins = 1
# Set the detector which we should extract the information:
if extractDet is None:
if ringerOperation < 0:
extractDet = Detector.Calorimetry
elif ringerOperation is RingerOperation.L2Calo:
extractDet = Detector.Calorimetry
elif ringerOperation is RingerOperation.L2:
extractDet = Detector.Tracking
else:
extractDet = Detector.CaloAndTrack
else:
extractDet = Detector.retrieve( extractDet )
if etaBins is None: etaBins = npCurrent.fp_array([])
if type(etaBins) is list: etaBins=npCurrent.fp_array(etaBins)
if etBins is None: etBins = npCurrent.fp_array([])
if type(etBins) is list: etBins=npCurrent.fp_array(etBins)
if etBins.size:
etBins = etBins * 1000. # Put energy in MeV
nEtBins = len(etBins)-1
if nEtBins >= np.iinfo(npCurrent.scounter_dtype).max:
self._fatal(('Number of et bins (%d) is larger or equal than maximum '
'integer precision can hold (%d). Increase '
'TuningTools.coreDef.npCurrent scounter_dtype number of bytes.'), nEtBins,
np.iinfo(npCurrent.scounter_dtype).max)
# Flag that we are separating data through bins
useBins=True
useEtBins=True
self._debug('E_T bins enabled.')
if not type(ringConfig) is list and not type(ringConfig) is np.ndarray:
ringConfig = [ringConfig] * (len(etaBins) - 1) if etaBins.size else 1
if type(ringConfig) is list: ringConfig=npCurrent.int_array(ringConfig)
if not len(ringConfig):
self._fatal('Rings size must be specified.');
if etaBins.size:
nEtaBins = len(etaBins)-1
if nEtaBins >= np.iinfo(npCurrent.scounter_dtype).max:
self._fatal(('Number of eta bins (%d) is larger or equal than maximum '
'integer precision can hold (%d). Increase '
'TuningTools.coreDef.npCurrent scounter_dtype number of bytes.'), nEtaBins,
np.iinfo(npCurrent.scounter_dtype).max)
if len(ringConfig) != nEtaBins:
self._fatal(('The number of rings configurations (%r) must be equal than '
'eta bins (%r) region config'),ringConfig, etaBins)
useBins=True
useEtaBins=True
self._debug('eta bins enabled.')
else:
self._debug('eta/et bins disabled.')
### Prepare to loop:
# Open root file
t = ROOT.TChain(treePath)
for inputFile in progressbar(fList, len(fList),
logger = self._logger,
prefix = "Creating collection tree "):
# Check if file exists
f = ROOT.TFile.Open(inputFile, 'read')
if not f or f.IsZombie():
self._warning('Couldn''t open file: %s', inputFile)
continue
# Inform user whether TTree exists, and which options are available:
self._debug("Adding file: %s", inputFile)
obj = f.Get(treePath)
if not obj:
self._warning("Couldn't retrieve TTree (%s)!", treePath)
self._info("File available info:")
f.ReadAll()
f.ReadKeys()
f.ls()
continue
elif not isinstance(obj, ROOT.TTree):
self._fatal("%s is not an instance of TTree!", treePath, ValueError)
t.Add( inputFile )
# Turn all branches off.
t.SetBranchStatus("*", False)
# RingerPhysVal hold the address of required branches
event = ROOT.RingerPhysVal()
# Add offline branches, these are always needed
cPos = 0
for var in __offlineBranches:
self.__setBranchAddress(t,var,event)
# Add online branches if using Trigger
if ringerOperation > 0:
for var in __onlineBranches:
self.__setBranchAddress(t,var,event)
## Allocating memory for the number of entries
entries = t.GetEntries()
nobs = entries if (nClusters is None or nClusters > entries or nClusters < 1) \
else nClusters
## Retrieve the dependent operation variables:
if useEtBins:
etBranch = 'el_et' if ringerOperation < 0 else 'trig_L2_calo_et'
self.__setBranchAddress(t,etBranch,event)
self._debug("Added branch: %s", etBranch)
if not getRatesOnly:
npEt = npCurrent.scounter_zeros(shape=npCurrent.shape(npat = 1, nobs = nobs))
self._debug("Allocated npEt with size %r", npEt.shape)
if useEtaBins:
etaBranch = "el_eta" if ringerOperation < 0 else "trig_L2_calo_eta"
self.__setBranchAddress(t,etaBranch,event)
self._debug("Added branch: %s", etaBranch)
if not getRatesOnly:
npEta = npCurrent.scounter_zeros(shape=npCurrent.shape(npat = 1, nobs = nobs))
self._debug("Allocated npEta with size %r", npEta.shape)
# The base information holder, such as et, eta and pile-up
if pileupRef is NotSet:
if ringerOperation > 0:
pileupRef = PileupReference.avgmu
else:
pileupRef = PileupReference.nvtx
pileupRef = PileupReference.retrieve( pileupRef )
self._info("Using '%s' as pile-up reference.", PileupReference.tostring( pileupRef ) )
if pileupRef is PileupReference.nvtx:
pileupBranch = 'el_nPileupPrimaryVtx'
pileupDataType = np.uint16
elif pileupRef is PileupReference.avgmu:
pileupBranch = 'avgmu'
pileupDataType = np.float32
else:
raise NotImplementedError("Pile-up reference %r is not implemented." % pileupRef)
baseInfoBranch = BaseInfo((etBranch, etaBranch, pileupBranch, 'el_phi' if ringerOperation < 0 else 'trig_L2_el_phi',),
(npCurrent.fp_dtype, npCurrent.fp_dtype, npCurrent.fp_dtype, pileupDataType) )
baseInfo = [None, ] * baseInfoBranch.nInfo
# Make sure all baseInfoBranch information is available:
for idx in baseInfoBranch:
self.__setBranchAddress(t,baseInfoBranch.retrieveBranch(idx),event)
# Allocate numpy to hold as many entries as possible:
if not getRatesOnly:
# Retrieve the rings information depending on ringer operation
ringerBranch = "el_ringsE" if ringerOperation < 0 else \
"trig_L2_calo_rings"
self.__setBranchAddress(t,ringerBranch,event)
if ringerOperation > 0:
if ringerOperation is RingerOperation.L2:
for var in __l2trackBranches:
self.__setBranchAddress(t,var,event)
if standardCaloVariables:
if ringerOperation in (RingerOperation.L2, RingerOperation.L2Calo,):
for var in __l2stdCaloBranches:
self.__setBranchAddress(t, var, event)
else:
self._warning("Unknown standard calorimeters for Operation:%s. Setting operation back to use rings variables.",
RingerOperation.tostring(ringerOperation))
t.GetEntry(0)
npat = 0
if extractDet in (Detector.Calorimetry,
Detector.CaloAndTrack,
Detector.All):
if standardCaloVariables:
npat+= 6
else:
npat += ringConfig.max()
if extractDet in (Detector.Tracking,
Detector.CaloAndTrack,
Detector.All):
if ringerOperation is RingerOperation.L2:
if useTRT:
self._info("Using TRT information!")
npat += 2
__l2trackBranches.append('trig_L2_el_nTRTHits')
__l2trackBranches.append('trig_L2_el_nTRTHiThresholdHits')
npat += 3
for var in __l2trackBranches:
self.__setBranchAddress(t,var,event)
self.__setBranchAddress(t,"trig_L2_el_pt",event)
elif ringerOperation < 0: # Offline
self._warning("Still need to implement tracking for the ringer offline.")
npPatterns = npCurrent.fp_zeros( shape=npCurrent.shape(npat=npat, #getattr(event, ringerBranch).size()
nobs=nobs)
)
self._debug("Allocated npPatterns with size %r", npPatterns.shape)
# Add E_T, eta and luminosity information
npBaseInfo = [npCurrent.zeros( shape=npCurrent.shape(npat=1, nobs=nobs ), dtype=baseInfoBranch.dtype(idx) )
for idx in baseInfoBranch]
else:
npPatterns = npCurrent.fp_array([])
npBaseInfo = [deepcopy(npCurrent.fp_array([])) for _ in baseInfoBranch]
## Allocate the branch efficiency collectors:
if getRates:
if ringerOperation < 0:
benchmarkDict = OrderedDict(
[( RingerOperation.Offline_CutBased_Loose , 'el_loose' ),
( RingerOperation.Offline_CutBased_Medium , 'el_medium' ),
( RingerOperation.Offline_CutBased_Tight , 'el_tight' ),
( RingerOperation.Offline_LH_Loose , 'el_lhLoose' ),
( RingerOperation.Offline_LH_Medium , 'el_lhMedium' ),
( RingerOperation.Offline_LH_Tight , 'el_lhTight' ),
])
else:
benchmarkDict = OrderedDict(
[( RingerOperation.L2Calo , 'trig_L2_calo_accept' ),
( RingerOperation.L2 , 'trig_L2_el_accept' ),
( RingerOperation.EFCalo , 'trig_EF_calo_accept' ),
( RingerOperation.HLT , 'trig_EF_el_accept' ),
])
from TuningTools.CreateData import BranchEffCollector, BranchCrossEffCollector
branchEffCollectors = OrderedDict()
branchCrossEffCollectors = OrderedDict()
for key, val in benchmarkDict.iteritems():
branchEffCollectors[key] = list()
branchCrossEffCollectors[key] = list()
# Add efficincy branch:
if getRates or getRatesOnly:
self.__setBranchAddress(t,val,event)
for etBin in range(nEtBins):
if useBins:
branchEffCollectors[key].append(list())
branchCrossEffCollectors[key].append(list())
for etaBin in range(nEtaBins):
etBinArg = etBin if useBins else -1
etaBinArg = etaBin if useBins else -1
argList = [ RingerOperation.tostring(key), val, etBinArg, etaBinArg ]
branchEffCollectors[key][etBin].append(BranchEffCollector( *argList ) )
if crossVal:
branchCrossEffCollectors[key][etBin].append(BranchCrossEffCollector( entries, crossVal, *argList ) )
# etBin
# etaBin
# benchmark dict
if self._logger.isEnabledFor( LoggingLevel.DEBUG ):
self._debug( 'Retrieved following branch efficiency collectors: %r',
[collector[0].printName for collector in traverse(branchEffCollectors.values())])
# end of (getRates)
etaBin = 0; etBin = 0
step = int(entries/100) if int(entries/100) > 0 else 1
## Start loop!
self._info("There is available a total of %d entries.", entries)
for entry in progressbar(range(entries), entries,
step = step, logger = self._logger,
prefix = "Looping over entries "):
#self._verbose('Processing eventNumber: %d/%d', entry, entries)
t.GetEntry(entry)
# Check if it is needed to remove energy regions (this means that if not
# within this range, it will be ignored as well for efficiency measuremnet)
if event.el_et < offEtCut:
self._verbose("Ignoring entry due to offline E_T cut.")
continue
# Add et distribution for all events
if not monitoring is None:
# Book all distribtions before the event selection
self.__fillHistograms(monitoring,filterType,event,False)
if ringerOperation > 0:
# Remove events which didn't pass L1_calo
if not supportTriggers and not event.trig_L1_accept:
#self._verbose("Ignoring entry due to L1Calo cut (trig_L1_accept = %r).", event.trig_L1_accept)
continue
if event.trig_L1_emClus < l1EmClusCut:
#self._verbose("Ignoring entry due to L1Calo E_T cut (%d < %r).", event.trig_L1_emClus, l1EmClusCut)
continue
if event.trig_L2_calo_et < l2EtCut:
#self._verbose("Ignoring entry due to L2Calo E_T cut.")
continue
if efEtCut is not None and event.trig_L2_calo_accept :
# EF calo is a container, search for electrons objects with et > cut
trig_EF_calo_et_list = stdvector_to_list(event.trig_EF_calo_et)
found=False
for v in trig_EF_calo_et_list:
if v < efEtCut: found=True
if found:
#self._verbose("Ignoring entry due to EFCalo E_T cut.")
continue
# Set discriminator target:
target = Target.Unknown
if reference is Reference.Truth:
if event.mc_isElectron and event.mc_hasZMother:
target = Target.Signal
elif not (event.mc_isElectron and (event.mc_hasZMother or event.mc_hasWMother) ):
target = Target.Background
elif reference is Reference.Off_Likelihood:
if event.el_lhTight: target = Target.Signal
elif not event.el_lhLoose: target = Target.Background
elif reference is Reference.AcceptAll:
target = Target.Signal if filterType is FilterType.Signal else Target.Background
else:
if event.el_tight: target = Target.Signal
elif not event.el_loose: target = Target.Background
# Run filter if it is defined
if filterType and \
( (filterType is FilterType.Signal and target != Target.Signal) or \
(filterType is FilterType.Background and target != Target.Background) or \
(target == Target.Unknown) ):
#self._verbose("Ignoring entry due to filter cut.")
continue
# Add et distribution for all events
if not monitoring is None:
# Book all distributions after the event selection
self.__fillHistograms(monitoring,filterType,event,True)
# Retrieve base information:
for idx in baseInfoBranch:
lInfo = getattr(event, baseInfoBranch.retrieveBranch(idx))
baseInfo[idx] = lInfo
if not getRatesOnly: npBaseInfo[idx][cPos] = lInfo
# Retrieve dependent operation region
if useEtBins:
etBin = self.__retrieveBinIdx( etBins, baseInfo[0] )
if useEtaBins:
etaBin = self.__retrieveBinIdx( etaBins, np.fabs( baseInfo[1]) )
# Check if bin is within range (when not using bins, this will always be true):
if (etBin < nEtBins and etaBin < nEtaBins):
# Retrieve patterns:
if not getRatesOnly:
if useEtBins: npEt[cPos] = etBin
if useEtaBins: npEta[cPos] = etaBin
## Retrieve calorimeter information:
cPat = 0
caloAvailable = True
if extractDet in (Detector.Calorimetry,
Detector.CaloAndTrack,
Detector.All):
if standardCaloVariables:
patterns = []
if ringerOperation is RingerOperation.L2Calo:
from math import cosh
cosh_eta = cosh( event.trig_L2_calo_eta )
# second layer ratio between 3x7 7x7
rEta = event.trig_L2_calo_e237 / event.trig_L2_calo_e277
base = event.trig_L2_calo_emaxs1 + event.trig_L2_calo_e2tsts1
# Ratio between first and second highest energy cells
eRatio = ( event.trig_L2_calo_emaxs1 - event.trig_L2_calo_e2tsts1 ) / base if base > 0 else 0
# ratio of energy in the first layer (hadronic particles should leave low energy)
F1 = event.trig_L2_calo_fracs1 / ( event.trig_L2_calo_et * cosh_eta )
# weta2 is calculated over the middle layer using 3 x 5
weta2 = event.trig_L2_calo_weta2
# wstot is calculated over the first layer using (typically) 20 strips
wstot = event.trig_L2_calo_wstot
# ratio between EM cluster and first hadronic layers:
Rhad1 = ( event.trig_L2_calo_ehad1 / cosh_eta ) / event.trig_L2_calo_et
# allocate patterns:
patterns = [rEta, eRatio, F1, weta2, wstot, Rhad1]
for pat in patterns:
npPatterns[npCurrent.access( pidx=cPat, oidx=cPos) ] = pat
cPat += 1
# end of ringerOperation
else:
# Remove events without rings
if getattr(event,ringerBranch).empty():
caloAvailable = False
# Retrieve rings:
if caloAvailable:
try:
patterns = stdvector_to_list( getattr(event,ringerBranch) )
lPat = len(patterns)
if lPat == ringConfig[etaBin]:
npPatterns[npCurrent.access(pidx=slice(cPat,ringConfig[etaBin]),oidx=cPos)] = patterns
else:
oldEtaBin = etaBin
if etaBin > 0 and ringConfig[etaBin - 1] == lPat:
etaBin -= 1
elif etaBin + 1 < len(ringConfig) and ringConfig[etaBin + 1] == lPat:
etaBin += 1
npPatterns[npCurrent.access(pidx=slice(cPat, ringConfig[etaBin]),oidx=cPos)] = patterns
self._warning(("Recovered event which should be within eta bin (%d: %r) "
"but was found to be within eta bin (%d: %r). "
"Its read eta value was of %f."),
oldEtaBin, etaBins[oldEtaBin:oldEtaBin+2],
etaBin, etaBins[etaBin:etaBin+2],
np.fabs( getattr(event,etaBranch)))
except ValueError:
self._logger.error(("Patterns size (%d) do not match expected "
"value (%d). This event eta value is: %f, and ringConfig is %r."),
lPat, ringConfig[etaBin], np.fabs( getattr(event,etaBranch)), ringConfig
)
continue
else:
if extractDet is Detector.Calorimetry:
# Also display warning when extracting only calorimetry!
self._warning("Rings not available")
continue
self._warning("Rings not available")
continue
cPat += ringConfig.max()
# which calo variables
# end of (extractDet needed calorimeter)
# And track information:
if extractDet in (Detector.Tracking,
Detector.CaloAndTrack,
Detector.All):
if caloAvailable or extractDet is Detector.Tracking:
if ringerOperation is RingerOperation.L2:
# Retrieve nearest deta/dphi only, so we need to find each one is the nearest:
if event.trig_L2_el_trkClusDeta.size():
clusDeta = npCurrent.fp_array( stdvector_to_list( event.trig_L2_el_trkClusDeta ) )
clusDphi = npCurrent.fp_array( stdvector_to_list( event.trig_L2_el_trkClusDphi ) )
bestTrackPos = int( np.argmin( clusDeta**2 + clusDphi**2 ) )
for var in __l2trackBranches:
npPatterns[npCurrent.access( pidx=cPat,oidx=cPos) ] = getattr(event, var)[bestTrackPos]
cPat += 1
else:
#self._verbose("Ignoring entry due to track information not available.")
continue
#for var in __l2trackBranches:
# npPatterns[npCurrent.access( pidx=cPat,oidx=cPos) ] = np.nan
# cPat += 1
elif ringerOperation < 0: # Offline
pass
# caloAvailable or only tracking
# end of (extractDet needs tracking)
# end of (getRatesOnly)
## Retrieve rates information:
if getRates:
for branch in branchEffCollectors.itervalues():
if not useBins:
branch.update(event)
else:
branch[etBin][etaBin].update(event)
if crossVal:
for branchCross in branchCrossEffCollectors.itervalues():
if not useBins:
branchCross.update(event)
else:
branchCross[etBin][etaBin].update(event)
# end of (getRates)
# We only increment if this cluster will be computed
cPos += 1
# end of (et/eta bins)
# Limit the number of entries to nClusters if desired and possible:
if not nClusters is None and cPos >= nClusters:
break
# for end
## Treat the rings information
if not getRatesOnly:
## Remove not filled reserved memory space:
if npPatterns.shape[npCurrent.odim] > cPos:
npPatterns = np.delete( npPatterns, slice(cPos,None), axis = npCurrent.odim)
## Segment data over bins regions:
# Also remove not filled reserved memory space:
if useEtBins:
npEt = npCurrent.delete( npEt, slice(cPos,None))
if useEtaBins:
npEta = npCurrent.delete( npEta, slice(cPos,None))
# Treat
npObject = self.treatNpInfo(cPos, npEt, npEta, useEtBins, useEtaBins,
nEtBins, nEtaBins, standardCaloVariables, ringConfig,
npPatterns, )
data = [self.treatNpInfo(cPos, npEt, npEta, useEtBins, useEtaBins,
nEtBins, nEtaBins, standardCaloVariables, ringConfig,
npData) for npData in npBaseInfo]
npBaseInfo = npCurrent.array( data, dtype=np.object )
else:
npObject = npCurrent.array([], dtype=npCurrent.dtype)
# not getRatesOnly
if getRates:
if crossVal:
for etBin in range(nEtBins):
for etaBin in range(nEtaBins):
for branchCross in branchCrossEffCollectors.itervalues():
if not useBins:
branchCross.finished()
else:
branchCross[etBin][etaBin].finished()
# Print efficiency for each one for the efficiency branches analysed:
for etBin in range(nEtBins) if useBins else range(1):
for etaBin in range(nEtaBins) if useBins else range(1):
for branch in branchEffCollectors.itervalues():
lBranch = branch if not useBins else branch[etBin][etaBin]
self._info('%s',lBranch)
if crossVal:
for branchCross in branchCrossEffCollectors.itervalues():
lBranchCross = branchCross if not useBins else branchCross[etBin][etaBin]
lBranchCross.dump(self._debug, printSort = True,
sortFcn = self._verbose)
# for branch
# for eta
# for et
# end of (getRates)
outputs = []
#if not getRatesOnly:
outputs.extend((npObject, npBaseInfo))
#if getRates:
outputs.extend((branchEffCollectors, branchCrossEffCollectors))
#outputs = tuple(outputs)
return outputs
from zipfile import BadZipfile
from copy import deepcopy
for f in files:
logger.info("Turning numpy matrix file '%s' into pre-processing file...", f)
fileparts = f.split("/")
folder = "/".join(fileparts[0:-1]) + "/"
fname = fileparts[-1]
try:
data = dict(load(f))
except BadZipfile, e:
logger.warning("Couldn't load file '%s'. Reason:\n%s", f, str(e))
continue
logger.debug("Finished loading file '%s'...", f)
for key in data:
if key == "W":
ppCol = deepcopy(data["W"])
from TuningTools.PreProc import *
for obj, idx, parent, _, _ in traverse(ppCol, tree_types=(np.ndarray,), max_depth=3):
parent[idx] = PreProcChain(RemoveMean(), Projection(matrix=obj), UnitaryRMS())
# Turn arrays into mutable objects:
ppCol = ppCol.tolist()
ppCol = fixPPCol(ppCol, len(ppCol[0][0]), len(ppCol[0]), len(ppCol))
if fname.endswith(".npz"):
fname = fname[:-4]
newFilePath = folder + fname + ".pic"
logger.info('Saving to: "%s"...', newFilePath)
place = PreProcArchieve(newFilePath, ppCol=ppCol).save(compress=False)
logger.info("File saved at path: '%s'", place)
def __call__(self, fList, ringerOperation, **kw):
"""
Read ntuple and return patterns and efficiencies.
Arguments:
- fList: The file path or file list path. It can be an argument list of
two types:
o List: each element is a string path to the file;
o Comma separated string: each path is separated via a comma
o Folders: Expand folders recursively adding also files within them to analysis
- ringerOperation: Set Operation type. It can be both a string or the
RingerOperation
Optional arguments:
- filterType [None]: whether to filter. Use FilterType enumeration
- reference [Truth]: set reference for targets. Use Reference enumeration
- treePath [Set using operation]: set tree name on file, this may be set to
use different sources then the default.
Default for:
o Offline: Offline/Egamma/Ntuple/electron
o L2: Trigger/HLT/Egamma/TPNtuple/e24_medium_L1EM18VH
- l1EmClusCut [None]: Set L1 cluster energy cut if operating on the trigger
- l2EtCut [None]: Set L2 cluster energy cut value if operating on the trigger
- offEtCut [None]: Set Offline cluster energy cut value
- nClusters [None]: Read up to nClusters. Use None to run for all clusters.
- getRatesOnly [False]: Read up to nClusters. Use None to run for all clusters.
- etBins [None]: E_T bins (GeV) where the data should be segmented
- etaBins [None]: eta bins where the data should be segmented
- ringConfig [100]: A list containing the number of rings available in the data
for each eta bin.
- crossVal [None]: Whether to measure benchmark efficiency splitting it
by the crossVal-validation datasets
- extractDet [None]: Which detector to export (use Detector enumeration).
Defaults are:
o L2Calo: Calorimetry
o L2: Tracking
o Offline: Calorimetry
o Others: CaloAndTrack
- standardCaloVariables [False]: Whether to extract standard track variables.
- useTRT [False]: Whether to export TRT information when dumping track
variables.
- supportTriggers [True]: Whether reading data comes from support triggers
"""
__eventBranches = [
'EventNumber', 'RunNumber', 'RandomRunNumber', 'MCChannelNumber',
'RandomLumiBlockNumber', 'MCPileupWeight', 'VertexZPosition',
'Zcand_M', 'Zcand_pt', 'Zcand_eta', 'Zcand_phi', 'Zcand_y',
'isTagTag'
]
__trackBranches = [
'elCand2_deltaeta1', 'elCand2_DeltaPOverP',
'elCand2_deltaphiRescaled', 'elCand2_d0significance',
'elCand2_trackd0pvunbiased', 'elCand2_eProbabilityHT'
]
__monteCarloBranches = [
'type',
'origin',
'originbkg',
'typebkg',
'isTruthElectronFromZ',
'TruthParticlePdgId',
'firstEgMotherPdgId',
'TruthParticleBarcode',
'firstEgMotherBarcode',
'MotherPdgId',
'MotherBarcode',
'FirstEgMotherTyp',
'FirstEgMotherOrigin',
'dRPdgId',
]
__onlineBranches = ['match', 'ringerMatch', 'ringer_rings']
__offlineBranches = ['et', 'eta']
# The current pid map used as offline reference
pidConfigs = {
key: value
for key, value in RingerOperation.efficiencyBranches().iteritems()
if key in (RingerOperation.Offline_LH_Tight,
RingerOperation.Offline_LH_Medium,
RingerOperation.Offline_LH_Loose,
RingerOperation.Offline_LH_VeryLoose)
}
# Retrieve information from keyword arguments
filterType = retrieve_kw(kw, 'filterType', FilterType.DoNotFilter)
reference = retrieve_kw(kw, 'reference', Reference.AcceptAll)
offEtCut = retrieve_kw(kw, 'offEtCut', None)
l2EtCut = retrieve_kw(kw, 'l2EtCut', None)
treePath = retrieve_kw(kw, 'treePath', 'ZeeCandidate')
nClusters = retrieve_kw(kw, 'nClusters', None)
etBins = retrieve_kw(kw, 'etBins', None)
etaBins = retrieve_kw(kw, 'etaBins', None)
crossVal = retrieve_kw(kw, 'crossVal', None)
ringConfig = retrieve_kw(kw, 'ringConfig', 100)
monitoring = retrieve_kw(kw, 'monitoring', None)
pileupRef = retrieve_kw(kw, 'pileupRef', NotSet)
getRates = retrieve_kw(kw, 'getRates', True)
getRatesOnly = retrieve_kw(kw, 'getRatesOnly', False)
getTagsOnly = retrieve_kw(kw, 'getTagsOnly', False)
extractDet = retrieve_kw(kw, 'extractDet', None)
import ROOT
#gROOT.ProcessLine (".x $ROOTCOREDIR/scripts/load_packages.C");
#ROOT.gROOT.Macro('$ROOTCOREDIR/scripts/load_packages.C')
if ROOT.gSystem.Load('libTuningTools') < 0:
self._fatal("Could not load TuningTools library", ImportError)
if 'level' in kw: self.level = kw.pop('level')
# and delete it to avoid mistakes:
checkForUnusedVars(kw, self._warning)
del kw
### Parse arguments
# Also parse operation, check if its type is string and if we can
# transform it to the known operation enum:
fList = csvStr2List(fList)
fList = expandFolders(fList)
ringerOperation = RingerOperation.retrieve(ringerOperation)
reference = Reference.retrieve(reference)
# Offline E_T cut
if offEtCut:
offEtCut = 1000. * offEtCut # Put energy in MeV
# Check whether using bins
useBins = False
useEtBins = False
useEtaBins = False
nEtaBins = 1
nEtBins = 1
if etaBins is None: etaBins = npCurrent.fp_array([])
if type(etaBins) is list: etaBins = npCurrent.fp_array(etaBins)
if etBins is None: etBins = npCurrent.fp_array([])
if type(etBins) is list: etBins = npCurrent.fp_array(etBins)
if etBins.size:
etBins = etBins * 1000. # Put energy in MeV
nEtBins = len(etBins) - 1
if nEtBins >= np.iinfo(npCurrent.scounter_dtype).max:
self._fatal((
'Number of et bins (%d) is larger or equal than maximum '
'integer precision can hold (%d). Increase '
'TuningTools.coreDef.npCurrent scounter_dtype number of bytes.'
), nEtBins,
np.iinfo(npCurrent.scounter_dtype).max)
# Flag that we are separating data through bins
useBins = True
useEtBins = True
self._debug('E_T bins enabled.')
if not type(ringConfig) is list and not type(ringConfig) is np.ndarray:
ringConfig = [ringConfig] * (len(etaBins) -
1) if etaBins.size else 1
if type(ringConfig) is list:
ringConfig = npCurrent.int_array(ringConfig)
if not len(ringConfig):
self._fatal('Rings size must be specified.')
if etaBins.size:
nEtaBins = len(etaBins) - 1
if nEtaBins >= np.iinfo(npCurrent.scounter_dtype).max:
self._fatal((
'Number of eta bins (%d) is larger or equal than maximum '
'integer precision can hold (%d). Increase '
'TuningTools.coreDef.npCurrent scounter_dtype number of bytes.'
), nEtaBins,
np.iinfo(npCurrent.scounter_dtype).max)
if len(ringConfig) != nEtaBins:
self._fatal((
'The number of rings configurations (%r) must be equal than '
'eta bins (%r) region config'), ringConfig, etaBins)
useBins = True
useEtaBins = True
self._debug('eta bins enabled.')
else:
self._debug('eta/et bins disabled.')
# The base information holder, such as et, eta and pile-up
if pileupRef is NotSet:
if ringerOperation > 0:
pileupRef = PileupReference.avgmu
else:
pileupRef = PileupReference.nvtx
pileupRef = PileupReference.retrieve(pileupRef)
self._info("Using '%s' as pile-up reference.",
PileupReference.tostring(pileupRef))
# Candidates: (1) is tags and (2) is probes. Default is probes
self._candIdx = 1 if getTagsOnly else 2
# Mutual exclusive arguments:
if not getRates and getRatesOnly:
self._logger.error(
"Cannot run with getRates set to False and getRatesOnly set to True. Setting getRates to True."
)
getRates = True
### Prepare to loop:
t = ROOT.TChain(treePath)
for inputFile in progressbar(fList,
len(fList),
logger=self._logger,
prefix="Creating collection tree "):
# Check if file exists
f = ROOT.TFile.Open(inputFile, 'read')
if not f or f.IsZombie():
self._warning('Couldn' 't open file: %s', inputFile)
continue
# Inform user whether TTree exists, and which options are available:
self._debug("Adding file: %s", inputFile)
obj = f.Get(treePath)
if not obj:
self._warning("Couldn't retrieve TTree (%s)!", treePath)
self._info("File available info:")
f.ReadAll()
f.ReadKeys()
f.ls()
continue
elif not isinstance(obj, ROOT.TTree):
self._fatal("%s is not an instance of TTree!", treePath,
ValueError)
t.Add(inputFile)
# Turn all branches off.
t.SetBranchStatus("*", False)
# RingerPhysVal hold the address of required branches
event = ROOT.SkimmedNtuple()
# Ready to retrieve the total number of events
t.GetEntry(0)
## Allocating memory for the number of entries
entries = t.GetEntries()
nobs = entries if (nClusters is None or nClusters > entries or nClusters < 1) \
else nClusters
## Retrieve the dependent operation variables:
if useEtBins:
etBranch = ('elCand%d_et') % (
self._candIdx) if ringerOperation < 0 else ('fcCand%d_et') % (
self._candIdx)
self.__setBranchAddress(t, etBranch, event)
self._debug("Added branch: %s", etBranch)
npEt = npCurrent.scounter_zeros(
shape=npCurrent.shape(npat=1, nobs=nobs))
self._debug("Allocated npEt with size %r", npEt.shape)
if useEtaBins:
etaBranch = ('elCand%d_eta') % (
self._candIdx) if ringerOperation < 0 else ('fcCand%d_eta') % (
self._candIdx)
self.__setBranchAddress(t, etaBranch, event)
self._debug("Added branch: %s", etaBranch)
npEta = npCurrent.scounter_zeros(
shape=npCurrent.shape(npat=1, nobs=nobs))
self._debug("Allocated npEta with size %r", npEta.shape)
if reference is Reference.Truth:
self.__setBranchAddress(t, ('elCand%d_isTruthElectronFromZ') %
(self._candIdx), event)
for var in __offlineBranches:
self.__setBranchAddress(t, ('elCand%d_%s') % (self._candIdx, var),
event)
#for var in pidConfigs.values():
# self.__setBranchAddress(t,var,event)
for var in __trackBranches:
self.__setBranchAddress(t, var, event)
# Add online branches if using Trigger
if ringerOperation > 0:
for var in __onlineBranches:
self.__setBranchAddress(t,
('fcCand%d_%s') % (self._candIdx, var),
event)
else:
self.__setBranchAddress(t, ('elCand%d_%s') %
(self._candIdx, 'ringer_rings'), event)
if pileupRef is PileupReference.nvtx:
pileupBranch = 'Nvtx'
pileupDataType = np.uint16
elif pileupRef is PileupReference.avgmu:
pileupBranch = 'averageIntPerXing'
pileupDataType = np.float32
else:
raise NotImplementedError(
"Pile-up reference %r is not implemented." % pileupRef)
#for var in __eventBranches +
for var in [pileupBranch]:
self.__setBranchAddress(t, var, event)
### Allocate memory
if extractDet == (Detector.Calorimetry):
npat = ringConfig.max()
elif extractDet == (Detector.Tracking):
npat = len(__trackBranches)
# NOTE: Check if pat is correct for both Calo and track data
elif extractDet in (Detector.CaloAndTrack, Detector.All):
npat = ringConfig.max() + len(__trackBranches)
npPatterns = npCurrent.fp_zeros(shape=npCurrent.shape(
npat=npat, #getattr(event, ringerBranch).size()
nobs=nobs))
self._debug("Allocated npPatterns with size %r", npPatterns.shape)
baseInfoBranch = BaseInfo(
(etBranch, etaBranch, pileupBranch),
(npCurrent.fp_dtype, npCurrent.fp_dtype, pileupDataType))
baseInfo = [
None,
] * baseInfoBranch.nInfo
# Add E_T, eta and luminosity information
npBaseInfo = [
npCurrent.zeros(shape=npCurrent.shape(npat=1, nobs=nobs),
dtype=baseInfoBranch.dtype(idx))
for idx in baseInfoBranch
]
from TuningTools.CreateData import BranchEffCollector, BranchCrossEffCollector
branchEffCollectors = OrderedDict()
branchCrossEffCollectors = OrderedDict()
if ringerOperation < 0:
from operator import itemgetter
benchmarkDict = OrderedDict(
sorted([(key, value) for key, value in
RingerOperation.efficiencyBranches().iteritems()
if key < 0 and not (isinstance(value, (list, tuple)))],
key=itemgetter(0)))
else:
benchmarkDict = OrderedDict()
for key, val in benchmarkDict.iteritems():
branchEffCollectors[key] = list()
branchCrossEffCollectors[key] = list()
# Add efficincy branch:
if ringerOperation < 0:
self.__setBranchAddress(t, val, event)
for etBin in range(nEtBins):
if useBins:
branchEffCollectors[key].append(list())
branchCrossEffCollectors[key].append(list())
for etaBin in range(nEtaBins):
etBinArg = etBin if useBins else -1
etaBinArg = etaBin if useBins else -1
argList = [
RingerOperation.tostring(key), val, etBinArg, etaBinArg
]
branchEffCollectors[key][etBin].append(
BranchEffCollector(*argList))
if crossVal:
branchCrossEffCollectors[key][etBin].append(
BranchCrossEffCollector(entries, crossVal,
*argList))
# etBin
# etaBin
# benchmark dict
if self._logger.isEnabledFor(LoggingLevel.DEBUG):
self._debug(
'Retrieved following branch efficiency collectors: %r', [
collector[0].printName
for collector in traverse(branchEffCollectors.values())
])
etaBin = 0
etBin = 0
step = int(entries / 100) if int(entries / 100) > 0 else 1
## Start loop!
self._info("There is available a total of %d entries.", entries)
cPos = 0
### Loop over entries
for entry in progressbar(range(entries),
entries,
step=step,
logger=self._logger,
prefix="Looping over entries "):
self._verbose('Processing eventNumber: %d/%d', entry, entries)
t.GetEntry(entry)
#print self.__getEt(event)
if event.elCand2_et < offEtCut:
self._debug(
"Ignoring entry due to offline E_T cut. E_T = %1.3f < %1.3f MeV",
event.elCand2_et, offEtCut)
continue
# Add et distribution for all events
if ringerOperation > 0:
if event.fcCand2_et < l2EtCut:
self._debug("Ignoring entry due Fast Calo E_T cut.")
continue
# Add et distribution for all events
# Set discriminator target:
target = Target.Unknown
# Monte Carlo cuts
if reference is Reference.Truth:
if getattr(event, ('elCand%d_isTruthElectronFromZ') %
(self._candIdx)):
target = Target.Signal
elif not getattr(event, ('elCand%d_isTruthElectronFromZ') %
(self._candIdx)):
target = Target.Background
# Offline Likelihood cuts
elif reference is Reference.Off_Likelihood:
if getattr(event,
pidConfigs[RingerOperation.Offline_LH_Tight]):
target = Target.Signal
elif not getattr(
event,
pidConfigs[RingerOperation.Offline_LH_VeryLoose]):
target = Target.Background
# By pass everything (Default)
elif reference is Reference.AcceptAll:
target = Target.Signal if filterType is FilterType.Signal else Target.Background
# Run filter if it is defined
if filterType and \
( (filterType is FilterType.Signal and target != Target.Signal) or \
(filterType is FilterType.Background and target != Target.Background) or \
(target == Target.Unknown) ):
#self._verbose("Ignoring entry due to filter cut.")
continue
## Retrieve base information and rings:
for idx in baseInfoBranch:
lInfo = getattr(event, baseInfoBranch.retrieveBranch(idx))
baseInfo[idx] = lInfo
# Retrieve dependent operation region
if useEtBins:
etBin = self.__retrieveBinIdx(etBins, baseInfo[0])
if useEtaBins:
etaBin = self.__retrieveBinIdx(etaBins, np.fabs(baseInfo[1]))
# Check if bin is within range (when not using bins, this will always be true):
if (etBin < nEtBins and etaBin < nEtaBins):
if useEtBins: npEt[cPos] = etBin
if useEtaBins: npEta[cPos] = etaBin
# Online operation
cPat = 0
caloAvailable = True
if ringerOperation > 0 and self.__get_ringer_onMatch(
event) < 1:
continue
# TODO Treat case where we don't use rings energy
# Check if the rings empty
if self.__get_rings_energy(event, ringerOperation).empty():
self._debug(
'No rings available in this event. Skipping...')
caloAvailable = False
# Retrieve rings:
if extractDet in (Detector.Calorimetry, Detector.CaloAndTrack,
Detector.All):
if caloAvailable:
try:
pass
patterns = stdvector_to_list(
self.__get_rings_energy(
event, ringerOperation))
lPat = len(patterns)
if lPat == ringConfig[etaBin]:
npPatterns[npCurrent.access(
pidx=slice(cPat, ringConfig[etaBin]),
oidx=cPos)] = patterns
else:
oldEtaBin = etaBin
if etaBin > 0 and ringConfig[etaBin -
1] == lPat:
etaBin -= 1
elif etaBin + 1 < len(
ringConfig) and ringConfig[etaBin +
1] == lPat:
etaBin += 1
npPatterns[npCurrent.access(
pidx=slice(cPat, ringConfig[etaBin]),
oidx=cPos)] = patterns
self._warning((
"Recovered event which should be within eta bin (%d: %r) "
"but was found to be within eta bin (%d: %r). "
"Its read eta value was of %f."),
oldEtaBin,
etaBins[oldEtaBin:oldEtaBin + 2],
etaBin,
etaBins[etaBin:etaBin + 2],
np.fabs(getattr(
event, etaBranch)))
except ValueError:
self._logger.error((
"Patterns size (%d) do not match expected "
"value (%d). This event eta value is: %f, and ringConfig is %r."
), lPat, ringConfig[etaBin],
np.fabs(
getattr(event, etaBranch)),
ringConfig)
continue
cPat += ringConfig[etaBin]
else:
# Also display warning when extracting only calorimetry!
self._warning("Rings not available")
continue
if extractDet in (Detector.Tracking, Detector.CaloAndTrack,
Detector.All):
for var in __trackBranches:
npPatterns[npCurrent.access(pidx=cPat,
oidx=cPos)] = getattr(
event, var)
if var == 'elCand2_eProbabilityHT':
from math import log
TRT_PID = npPatterns[npCurrent.access(pidx=cPat,
oidx=cPos)]
epsilon = 1e-99
if TRT_PID >= 1.0: TRT_PID = 1.0 - 1.e-15
elif TRT_PID <= 0.0: TRT_PID = epsilon
tau = 15.0
TRT_PID = -(1 / tau) * log((1.0 / TRT_PID) - 1.0)
npPatterns[npCurrent.access(pidx=cPat,
oidx=cPos)] = TRT_PID
cPat += 1
## Retrieve rates information:
if getRates and ringerOperation < 0:
#event.elCand2_isEMVerLoose2015 = not( event.elCand2_isEMVeryLoose2015 & 34896 )
event.elCand2_isEMLoose2015 = not (
event.elCand2_isEMLoose2015 & 34896)
event.elCand2_isEMMedium2015 = not (
event.elCand2_isEMMedium2015 & 276858960)
event.elCand2_isEMTight2015 = not (
event.elCand2_isEMTight2015 & 281053264)
for branch in branchEffCollectors.itervalues():
if not useBins:
branch.update(event)
else:
branch[etBin][etaBin].update(event)
if crossVal:
for branchCross in branchCrossEffCollectors.itervalues(
):
if not useBins:
branchCross.update(event)
else:
branchCross[etBin][etaBin].update(event)
# end of (getRates)
if not monitoring is None:
self.__fillHistograms(monitoring, filterType, pileupRef,
pidConfigs, event)
# We only increment if this cluster will be computed
cPos += 1
# end of (et/eta bins)
# Limit the number of entries to nClusters if desired and possible:
if not nClusters is None and cPos >= nClusters:
break
# for end
## Treat the rings information
## Remove not filled reserved memory space:
if npPatterns.shape[npCurrent.odim] > cPos:
npPatterns = np.delete(npPatterns,
slice(cPos, None),
axis=npCurrent.odim)
## Segment data over bins regions:
# Also remove not filled reserved memory space:
if useEtBins:
npEt = npCurrent.delete(npEt, slice(cPos, None))
if useEtaBins:
npEta = npCurrent.delete(npEta, slice(cPos, None))
# Treat
standardCaloVariables = False
npObject = self.treatNpInfo(
cPos,
npEt,
npEta,
useEtBins,
useEtaBins,
nEtBins,
nEtaBins,
standardCaloVariables,
ringConfig,
npPatterns,
)
data = [
self.treatNpInfo(cPos, npEt, npEta, useEtBins, useEtaBins, nEtBins,
nEtaBins, standardCaloVariables, ringConfig,
npData) for npData in npBaseInfo
]
npBaseInfo = npCurrent.array(data, dtype=np.object)
if getRates:
if crossVal:
for etBin in range(nEtBins):
for etaBin in range(nEtaBins):
for branchCross in branchCrossEffCollectors.itervalues(
):
if not useBins:
branchCross.finished()
else:
branchCross[etBin][etaBin].finished()
# Print efficiency for each one for the efficiency branches analysed:
for etBin in range(nEtBins) if useBins else range(1):
for etaBin in range(nEtaBins) if useBins else range(1):
for branch in branchEffCollectors.itervalues():
lBranch = branch if not useBins else branch[etBin][
etaBin]
self._info('%s', lBranch)
if crossVal:
for branchCross in branchCrossEffCollectors.itervalues(
):
lBranchCross = branchCross if not useBins else branchCross[
etBin][etaBin]
lBranchCross.dump(self._debug,
printSort=True,
sortFcn=self._verbose)
# for branch
# for eta
# for et
else:
branchEffCollectors = None
branchCrossEffCollectors = None
# end of (getRates)
outputs = []
outputs.extend((npObject, npBaseInfo))
if getRates:
outputs.extend((branchEffCollectors, branchCrossEffCollectors))
return outputs
refIdx = etIdx + 3
if etIdx == 0: # 15 up to 20
shorterEffTable[etIdx,etaIdx] = val[refIdx,etaIdx]
if etIdx == 1: # merge 20, 25
shorterEffTable[etIdx,etaIdx] = (val[refIdx,etaIdx]*.4 + val[refIdx+1,etaIdx]*.6)
if etIdx == 2: # merge 30, 35
shorterEffTable[etIdx,etaIdx] = (val[refIdx+1,etaIdx]*.48 + val[refIdx+2,etaIdx]*.52)
if etIdx == 3: # merge 40, 45
shorterEffTable[etIdx,etaIdx] = (val[refIdx+2,etaIdx]*.5 + val[refIdx+3,etaIdx]*.5)
return shorterEffTable
from RingerCore import traverse
pdrefs = mergeEffTable( medium20160701 )
print pdrefs
pfrefs = np.array( [[0.05]*len(etaBins)]*len(etBins) )*100. # 3 5 7 10
efficiencyValues = np.array([np.array([refs]) for refs in zip(traverse(pdrefs,tree_types=(np.ndarray),simple_ret=True)
,traverse(pfrefs,tree_types=(np.ndarray),simple_ret=True))]).reshape(pdrefs.shape + (2,) )
print efficiencyValues
basePath = '/home/wsfreund/CERN-DATA'
sgnInputFile = 'user.jodafons.mc15_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee.merge.AOD.e3601_s2876_r7917_r7676.dump.trigPB.p0200_GLOBAL/'
bkgInputFile = 'user.jodafons.mc15_13TeV.423300.Pythia8EvtGen_A14NNPDF23LO_perf_JF17.merge.AOD.e3848_s2876_r7917_r7676.dump.trigEL.p0201_GLOBAL/'
outputFile = 'mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.eg.std.grid.medium'
treePath = ["HLT/Egamma/Expert/support/probes",
"HLT/Egamma/Expert/support/trigger"]
#crossValPath = 'crossValid_5sorts.pic.gz'
#from TuningTools import CrossValidArchieve
#with CrossValidArchieve( crossValPath ) as CVArchieve:
# crossVal = CVArchieve
# del CVArchieve
,[ 0.95465, 0.94708108, 0.8706, 0.93477684] # Et 20
,[ 0.96871, 0.96318919, 0.87894, 0.95187642] # Et 30
,[ 0.97425, 0.97103378, 0.884, 0.96574474] # Et 40
,[ 0.97525, 0.97298649, 0.887, 0.96703158]])*100. # Et 50
veryloose20160701 = np.array(
[[ 0.978 , 0.96458108, 0.9145 , 0.95786316]
,[ 0.98615, 0.97850541, 0.9028 , 0.96738947]
,[ 0.99369, 0.9900427 , 0.90956, 0.97782105]
,[ 0.995 , 0.99293919, 0.917 , 0.98623421]
,[ 0.99525, 0.99318919, 0.9165 , 0.98582632]])*100.
from RingerCore import traverse
pdrefs = medium20160701
pfrefs = np.array( [[0.05]*len(etaBins)]*len(etBins) )*100.
efficiencyValues = np.array([np.array([refs]) for refs in zip(traverse(pdrefs,tree_types=(np.ndarray),simple_ret=True)
,traverse(pfrefs,tree_types=(np.ndarray),simple_ret=True))]).reshape(pdrefs.shape + (2,) )
basePath = '/home/jodafons/CERN-DATA/backup/old_samples/mc15_13TeV/'
sgnInputFile = 'user.jodafons.mc15_13TeV.361106.Zee.merge.SelectionZee.PhysVal.r0005_GLOBAL'
bkgInputFile = 'user.jodafons.mc15_13TeV.423300.JF17.merge.SelectionFakes.PhysVal.r0005_GLOBAL'
outputFile = 'mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.medium'
treePath = ["run_284500/HLT/Egamma/Expert/support/probes",
"run_284500/HLT/Egamma/Expert/support/fakes"]
#crossValPath = 'crossValid_5sorts.pic.gz'
#from TuningTools import CrossValidArchieve
#with CrossValidArchieve( crossValPath ) as CVArchieve:
# crossVal = CVArchieve
# del CVArchieve
def getModels(self, summaryInfoList, **kw):
refBenchCol = kw.pop( 'refBenchCol', None )
configCol = kw.pop( 'configCol', [] )
muBin = kw.pop( 'muBin', [-999,9999] )
checkForUnusedVars( kw, self._logger.warning )
# Treat the summaryInfoList
if not isinstance( summaryInfoList, (list,tuple)):
summaryInfoList = [ summaryInfoList ]
summaryInfoList = list(traverse(summaryInfoList,simple_ret=True))
nSummaries = len(summaryInfoList)
if not nSummaries:
logger.fatal("Summary dictionaries must be specified!")
if refBenchCol is None:
refBenchCol = summaryInfoList[0].keys()
# Treat the reference benchmark list
if not isinstance( refBenchCol, (list,tuple)):
refBenchCol = [ refBenchCol ] * nSummaries
if len(refBenchCol) == 1:
refBenchCol = refBenchCol * nSummaries
nRefs = len(list(traverse(refBenchCol,simple_ret=True)))
# Make sure that the lists are the same size as the reference benchmark:
nConfigs = len(list(traverse(configCol,simple_ret=True)))
if nConfigs == 0:
configCol = [None for i in range(nRefs)]
elif nConfigs == 1:
configCol = configCol * nSummaries
nConfigs = len(list(traverse(configCol,simple_ret=True)))
if nConfigs != nRefs:
logger.fatal("Summary size is not equal to the configuration list.", ValueError)
if nRefs == nConfigs == nSummaries:
# If user input data without using list on the configuration, put it as a list:
for o, idx, parent, _, _ in traverse(configCol):
parent[idx] = [o]
for o, idx, parent, _, _ in traverse(refBenchCol):
parent[idx] = [o]
configCol = list(traverse(configCol,max_depth_dist=1,simple_ret=True))
refBenchCol = list(traverse(refBenchCol,max_depth_dist=1,simple_ret=True))
nConfigs = len(configCol)
nSummary = len(refBenchCol)
if nRefs != nConfigs != nSummary:
logger.fatal("Number of references, configurations and summaries do not match!", ValueError)
discrList = []
from itertools import izip, count
for summaryInfo, refBenchmarkList, configList in zip(summaryInfoList,refBenchCol,configCol):
if type(summaryInfo) is str:
self._logger.info('Loading file "%s"...', summaryInfo)
summaryInfo = load(summaryInfo)
elif type(summaryInfo) is dict:
pass
else:
logger.fatal("Cross-valid summary info is not string and not a dictionary.", ValueError)
for idx, refBenchmarkName, config in izip(count(), refBenchmarkList, configList):
try:
key = filter(lambda x: refBenchmarkName in x, summaryInfo)[0]
refDict = summaryInfo[ key ]
except IndexError :
self._logger.fatal("Could not find reference %s in summaryInfo. Available options are: %r", refBenchmarkName, summaryInfo.keys())
self._logger.info("Using Reference key: %s", key )
ppInfo = summaryInfo['infoPPChain']
etBinIdx = refDict['etBinIdx']
etaBinIdx = refDict['etaBinIdx']
etBin = refDict['etBin']
etaBin = refDict['etaBin']
info = refDict['infoOpBest'] if config is None else refDict['config_' + str(config).zfill(3)]['infoOpBest']
# Check if user specified parameters for exporting discriminator
# operation information:
sort = info['sort']
init = info['init']
pyThres = info['cut']
from RingerCore import retrieveRawDict
if isinstance( pyThres, float ):
pyThres = RawThreshold( thres = pyThres
, etBinIdx = etBinIdx, etaBinIdx = etaBinIdx
, etBin = etBin, etaBin = etaBin)
else:
# Get the object from the raw dict
pyThres = retrieveRawDict( pyThres )
if pyThres.etBin is None:
pyThres.etBin = etBin
elif pyThres.etBin is '':
pyThres.etBin = etBin
elif isinstance( pyThres.etBin, (list,tuple)):
pyThres.etBin = np.array( pyThres.etBin)
if not(np.array_equal( pyThres.etBin, etBin )):
self._logger.fatal("etBin does not match for threshold! Should be %r, is %r", pyThres.etBin, etBin )
if pyThres.etaBin is None:
pyThres.etaBin = etaBin
elif pyThres.etaBin is '':
pyThres.etaBin = etaBin
elif isinstance( pyThres.etaBin, (list,tuple)):
pyThres.etaBin = np.array( pyThres.etaBin)
if not(np.array_equal( pyThres.etaBin, etaBin )):
self._logger.fatal("etaBin does not match for threshold! Should be %r, is %r", pyThres.etaBin, etaBin )
if type(pyThres) is RawThreshold:
thresValues = [pyThres.thres]
else:
thresValues = [pyThres.slope, pyThres.intercept, pyThres.rawThres]
pyPreProc = ppInfo['sort_'+str(sort).zfill(3)]['items'][0]
pyPreProc = retrieveRawDict( pyPreProc )
useCaloRings=False; useTrack=False; useShowerShape=False
if type(pyPreProc) is Norm1:
useCaloRings=True
elif type(pyPreProc) is TrackSimpleNorm:
useTrack=True
elif type(pyPreProc) is ShowerShapesSimpleNorm:
useShowerShape=True
elif type(pyPreProc) is ExpertNetworksSimpleNorm:
useCaloRings=True; useTrack=True
elif type(pyPreProc) is ExpertNetworksShowerShapeSimpleNorm:
useCaloRings=True; useShowerShape=True
elif type(pyPreProc) is ExpertNetworksShowerShapeAndTrackSimpleNorm:
useCaloRings=True; useTrack=True; useShowerShape=True
elif type(pyPreProc) is PreProcMerge:
for slot in pyPreProc.slots:
if type(pyPreProc) is Norm1:
useCaloRings=True
elif type(pyPreProc) is TrackSimpleNorm:
useTrack=True
elif type(pyPreProc) is ShowerShapesSimpleNorm:
useShowerShape=True
elif type(pyPreProc) is ExpertNetworksSimpleNorm:
useCaloRings=True; useTrack=True
elif type(pyPreProc) is ExpertNetworksShowerShapeSimpleNorm:
useCaloRings=True; useShowerShape=True
elif type(pyPreProc) is ExpertNetworksShowerShapeAndTrackSimpleNorm:
useCaloRings=True; useTrack=True; useShowerShape=True
else:
self._logger.fatal('PrepProc strategy not found...')
discrDict = info['discriminator']
model = {
'discriminator' : discrDict,
'threshold' : thresValues,
'etBin' : etBin,
'etaBin' : etaBin,
'muBin' : muBin,
'etBinIdx' : etBinIdx,
'etaBinIdx' : etaBinIdx,
}
removeOutputTansigTF = refDict.get('removeOutputTansigTF', None )
model['removeOutputTansigTF'] = removeOutputTansigTF
model['useCaloRings'] = useCaloRings
model['useShowerShape'] = useShowerShape
model['useTrack'] = useTrack
discrList.append( model )
self._logger.info('neuron = %d, sort = %d, init = %d',
info['neuron'],
info['sort'],
info['init'])
# for benchmark
# for summay in list
return discrList
from RingerCore import Logger, LoggingLevel, save, load, expandFolders, traverse
import numpy as np
from TuningTools.coreDef import retrieve_npConstants
npCurrent, _ = retrieve_npConstants()
npCurrent.level = args.output_level
logger = Logger.getModuleLogger( __name__, args.output_level )
files = expandFolders( args.inputs ) # FIXME *.npz
from zipfile import BadZipfile
for f in files:
logger.info("Changing representation of file '%s'...", f)
try:
data = dict(load(f))
except BadZipfile, e:
logger.warning("Couldn't load file '%s'. Reason:\n%s", f, str(e))
continue
logger.debug("Finished loading file '%s'...", f)
for key in data:
if key == 'W':
for obj, idx, parent, _, _ in traverse(data[key],
tree_types = (np.ndarray,),
max_depth = 3):
parent[idx] = obj.T
elif type(data[key]) is np.ndarray:
logger.debug("Checking key '%s'...", key)
data[key] = npCurrent.toRepr(data[key])
path = save(data, f, protocol = 'savez_compressed')
logger.info("Overwritten file '%s'",f)
from TuningTools.PreProc import *
from TuningTools import CrossValid
if args.pp_nEtaBins is NotSet:
raise NoBinInfo('eta')
if args.pp_nEtBins is NotSet:
raise NoBinInfo('et')
# Retrieve information:
import ast, re
replacer = re.compile("(\w+(\(.*?\))?)")
args.ppCol = replacer.sub(r'"\1"', args.ppCol)
ppCol = ast.literal_eval(args.ppCol)
from RingerCore import traverse
class_str_re = re.compile("(\w+)(\(.*?\))?")
fix_args = re.compile("(\([^{}]+)((?<!,)\))")
dict_args = re.compile("(\{.*\})")
for str_, idx, parent, _, _ in traverse(ppCol):
m = class_str_re.match(str_)
if m:
# The class representation in string:
class_str = m.group(1)
# Retrieve the class itself (must be from PreProc module)
tClass = str_to_class( "TuningTools.PreProc", class_str)
# Retrieve the arguments, if available
class_attr = m.group(2)
if class_attr:
# Fix the arguments:
m2 = fix_args.search( m.group(2) )
if m2:
class_attr = m2.expand(r'\1,\2')
# Parse it:
class_attr_parsed = ast.literal_eval( class_attr )
f)
fileparts = f.split('/')
folder = '/'.join(fileparts[0:-1]) + '/'
fname = fileparts[-1]
try:
data = dict(load(f))
except BadZipfile, e:
logger.warning("Couldn't load file '%s'. Reason:\n%s", f, str(e))
continue
logger.debug("Finished loading file '%s'...", f)
for key in data:
if key == 'W':
ppCol = deepcopy(data['W'])
from TuningTools.PreProc import *
for obj, idx, parent, _, _ in traverse(ppCol,
tree_types=(np.ndarray, ),
max_depth=3):
parent[idx] = PreProcChain(RemoveMean(),
Projection(matrix=obj),
UnitaryRMS())
# Turn arrays into mutable objects:
ppCol = ppCol.tolist()
ppCol = fixPPCol(ppCol, len(ppCol[0][0]), len(ppCol[0]),
len(ppCol))
if fname.endswith('.npz'):
fname = fname[:-4]
newFilePath = folder + fname + '.pic'
logger.info('Saving to: "%s"...', newFilePath)
place = PreProcArchieve(newFilePath, ppCol=ppCol).save(compress=False)
logger.info("File saved at path: '%s'", place)
args = parser.parse_args(namespace=LoggerNamespace())
from RingerCore import Logger, LoggingLevel, save, load, expandFolders, traverse
import numpy as np
from TuningTools.coreDef import npCurrent
npCurrent.level = args.output_level
logger = Logger.getModuleLogger(__name__, args.output_level)
files = expandFolders(args.inputs) # FIXME *.npz
from zipfile import BadZipfile
for f in files:
logger.info("Changing representation of file '%s'...", f)
try:
data = dict(load(f))
except BadZipfile, e:
logger.warning("Couldn't load file '%s'. Reason:\n%s", f, str(e))
continue
logger.debug("Finished loading file '%s'...", f)
for key in data:
if key == 'W':
for obj, idx, parent, _, _ in traverse(data[key],
tree_types=(np.ndarray, ),
max_depth=3):
parent[idx] = obj.T
elif type(data[key]) is np.ndarray:
logger.debug("Checking key '%s'...", key)
data[key] = npCurrent.toRepr(data[key])
path = save(data, f, protocol='savez_compressed')
logger.info("Overwritten file '%s'", f)
args.pp_nEtaBins = 1
args.pp_nEtBins = 1
elif args.pp_nEtBins is NotSet:
raise NoBinInfo('et')
elif args.pp_nEtaBins is NotSet:
raise NoBinInfo('eta')
# Retrieve information:
import ast, re
replacer = re.compile("(\w+(\(.*?\))?)")
args.ppCol = replacer.sub(r'"\1"', args.ppCol)
ppCol = ast.literal_eval(args.ppCol)
from RingerCore import traverse
class_str_re = re.compile("(\w+)(\(.*?\))?")
fix_args = re.compile("(\([^{}]+)((?<!,)\))")
dict_args = re.compile("(\{.*\})")
for str_, idx, parent, _, _ in traverse(ppCol):
m = class_str_re.match(str_)
if m:
# The class representation in string:
class_str = m.group(1)
# Retrieve the class itself (must be from PreProc module)
tClass = str_to_class("TuningTools.PreProc", class_str)
# Retrieve the arguments, if available
class_attr = m.group(2)
if class_attr:
# Fix the arguments:
m2 = fix_args.search(m.group(2))
if m2:
class_attr = m2.expand(r'\1,\2')
# Parse it:
class_attr_parsed = ast.literal_eval(class_attr)
def expandFolders(pathList, filters=None, logger=None, level=None):
"""
Expand all folders to the contained files using the filters on pathList
Input arguments:
-> pathList: a list containing paths to files and folders;
filters;
-> filters: return a list for each filter with the files contained on the
list matching the filter glob.
-> logger: whether to print progress using logger;
-> level: logging level to print messages with logger;
WARNING: This function is extremely slow and will severely decrease
performance if used to expand base paths with several folders in it.
"""
if not isinstance(pathList, (
list,
tuple,
)):
pathList = [pathList]
from glob import glob
if filters is None:
filters = ['*']
if not (type(filters) in (
list,
tuple,
)):
filters = [filters]
retList = [[] for idx in range(len(filters))]
from RingerCore import progressbar, traverse
pathList = list(
traverse([
glob(path) if '*' in path else path
for path in traverse(pathList, simple_ret=True)
],
simple_ret=True))
for path in progressbar(pathList,
len(pathList),
'Expanding folders: ',
60,
50,
True if logger is not None else False,
logger=logger,
level=level):
path = expandPath(path)
if not os.path.exists(path):
raise ValueError("Cannot reach path '%s'" % path)
if os.path.isdir(path):
for idx, filt in enumerate(filters):
cList = filter(lambda x: not (os.path.isdir(x)),
[f for f in glob(os.path.join(path, filt))])
if cList:
retList[idx].extend(cList)
folders = [
os.path.join(path, f) for f in os.listdir(path)
if os.path.isdir(os.path.join(path, f))
]
if folders:
recList = expandFolders(folders, filters)
if len(filters) is 1:
recList = [recList]
for l in recList:
retList[idx].extend(l)
else:
for idx, filt in enumerate(filters):
if path in glob(os.path.join(os.path.dirname(path), filt)):
retList[idx].append(path)
if len(filters) is 1:
retList = retList[0]
return retList
