def find_threshold_best_sp(sgn_hist, noise_hist):
from RingerCore import calcSP
nbins = sgn_hist.GetNbinsX()
det_total = sgn_hist.Integral(0, nbins + 1)
noise_total = noise_hist.Integral(0, nbins + 1)
sp_max = -999
best_threshold = -999
best_det = -999
best_fa = -999
i = 0
while i < nbins + 1:
# Detection
det_passed = sgn_hist.Integral(i, nbins + 1)
# False alarm
noise_passed = noise_hist.Integral(i, nbins + 1)
det = det_passed / det_total
fa = noise_passed / noise_total
sp = calcSP(det, 1 - fa)
if sp > sp_max:
sp_max = sp
best_det = det
best_fa = fa
threshold = sgn_hist.GetBinCenter(i)
i += 1
return sp_max, best_det, best_fa, threshold
def __init__(self, name, ref, tobj, opObj, bobj): MonitoringOperationInfo.__init__(self, opObj) #name self._name = name self._perf = dict() self._ref = dict() #Retrieve information from benchmark self._ref['reference'] = bobj['reference'] self._ref['det'] = bobj['signal_efficiency']['efficiency'] self._ref['fa'] = bobj['background_efficiency']['efficiency'] #Calculate SP from reference self._ref['sp'] = calcSP(self._ref['det'], 100-self._ref['fa']) #Hold values for key in self._keys: self._perf[key] = tobj[key]*100
def getEffPoint( self, name, subset = [None, None], outputs = [None, None], pileup = [None,None], thres = None, makeCorr = True ):
from TuningTools.Neural import PerformancePoint
auc = self.rawPerf.auc if self.rawPerf else -1
if not isinstance(subset, (tuple,list)):
if subset is None:
if not(any([o is None for o in outputs])): self._fatal("Subset must be specified when outputs is used.")
subset = self.subset
subset = [CuratedSubset.tosgn(subset),CuratedSubset.tobkg(subset)]
else:
if len(subset) is 1:
if subset[0] is None:
if not(any([o is None for o in outputs])): self._fatal("Subset must be specified when outputs is used.")
subset = [self.subset]
subset = [CuratedSubset.tosgn(subset[0]),CuratedSubset.tobkg(subset[0])]
else:
if any([s is None for s in subset]):
if not(any([o is None for o in outputs])): self._fatal("Subset must be specified when outputs is used.")
subset = [self.subset, self.subset]
subset = [CuratedSubset.tosgn(subset[0]),CuratedSubset.tobkg(subset[1])]
self._effSubset = subset
if any([o is None for o in outputs]):
#if outputs[0] is None:
if isinstance(subset, (list,tuple)):
# FIXME This assumes that sgnOut is cached:
outputs = [(self.getOutput(CuratedSubset.topattern(s)) if CuratedSubset.tobinary(s) is not self.subset else o )
for o, s in zip([self.sgnOut,self.bkgOut], subset)]
else:
if CuratedSubset.tobinary(subset) is not self.subset:
outputs = self.getOutput(CuratedSubset.topattern(subset))
else:
outputs = [self.sgnOut,self.bkgOut]
# NOTE: This can be commented out to improve speed
try:
from libTuningToolsLib import genRoc
except ImportError:
from libTuningTools import genRoc
o = genRoc(outputs[0], outputs[1], +self._ol, -self._ol, 0.001 )
auc = Roc( o ).auc
self._effOutput = outputs
if thres is None: thres = self.thres if makeCorr else self.rawThres
pd = self._calcEff( subset[0], output = outputs[0], pileup = pileup[0], thres = thres, makeCorr = makeCorr )
pf = self._calcEff( subset[1], output = outputs[1], pileup = pileup[1], thres = thres, makeCorr = makeCorr )
sp = calcSP(pd, 1. - pf)
return PerformancePoint( name, sp, pd, pf, thres, perc = False, auc = auc
, etBinIdx = self.etBinIdx, etaBinIdx = self.etaBinIdx
, etBin = self.etBin, etaBin = self.etaBin )
def __init__(self, name, ref, tobj, opObj, bobj):
MonitoringOperationInfo.__init__(self, opObj)
#name
self._name = name
self._perf = dict()
self._ref = dict()
#Retrieve information from benchmark
self._ref['reference'] = bobj['reference']
try:
self._ref['det'] = bobj['signalEfficiency']['efficiency']
self._ref['fa'] = bobj['backgroundEfficiency']['efficiency']
except:
self._ref['det'] = bobj['signal_efficiency']['efficiency']
self._ref['fa'] = bobj['background_efficiency']['efficiency']
#Calculate SP from reference
self._ref['sp'] = calcSP(self._ref['det'], 100 - self._ref['fa'])
#Hold values
for key in self._keys:
self._perf[key] = tobj[key] * 100
def plot_rocs(plotObjects, kwargs):
from ROOT import kCyan, kRed, kGreen, kBlue, kBlack, kMagenta, kGray, kWhite, kYellow
Colors = [kBlue, kRed, kMagenta, kBlack, kCyan, kGreen]
from RingerCore import StdPair as std_pair
from util import line, minmax
dset = kwargs['set']
ref = kwargs['reference']
refVal = kwargs['refVal']
eps = kwargs['eps']
savename = kwargs['cname'] + '.pdf'
#Some protection
if not ('operation' in dset or 'tst' in dset):
raise ValueError('Option set must be: tst (test) or val (validation)')
if not ('SP' in ref or 'Pd' in ref or 'Pf' in ref):
raise ValueError('Option reference must be: SP, Pd or Pf')
#Create dict to hold all list plots
curves = dict()
#list of dicts to dict of lists
for name in plotObjects.keys():
curves[name] = plotObjects.tolist(name)
paintIdx = kwargs['paintListIdx'] # [best, worst]
paintCurves = [
std_pair(plotObjects.index_correction(paintIdx[0]), kBlack),
std_pair(plotObjects.index_correction(paintIdx[1]), kRed)
]
curves['roc'] = curves['roc_' + dset]
#Start to build all ROOT objects
from ROOT import TCanvas, gROOT, kTRUE
gROOT.SetBatch(kTRUE)
canvas = TCanvas('canvas', 'canvas', 1600, 1300)
x_limits = [0.00, 0.40]
y_limits = [0.6, 1.03]
#create dummy graph
dummy = curves['roc'][0]
dummy.SetTitle('Receive Operation Curve')
dummy.GetXaxis().SetTitle('False Alarm')
dummy.GetYaxis().SetTitle('Detection')
dummy.GetHistogram().SetAxisRange(y_limits[0], y_limits[1], 'Y')
dummy.GetHistogram().SetAxisRange(x_limits[0], x_limits[1], 'X')
dummy.Draw('AL')
corredor = None
target = None
from ROOT import TBox
if ref == 'Pf':
corredor = TBox(refVal - eps, y_limits[0], refVal + eps, y_limits[1])
target = line(refVal, y_limits[0], refVal, y_limits[1], kBlack, 2, 1,
'')
elif ref == 'Pd':
corredor = TBox(x_limits[0], refVal - eps, x_limits[1], refVal + eps)
target = line(x_limits[0], refVal, x_limits[1], refVal, kBlack, 2, 1,
'')
if ref != 'SP':
corredor.SetFillColor(kYellow - 9)
corredor.Draw('same')
target.Draw('same')
canvas.Modified()
canvas.Update()
#Plot curves
for c in curves['roc']:
c.SetLineColor(kGray + 1)
#c.SetMarkerStyle(7)
#c.SetMarkerColor(kBlue)
c.SetLineWidth(1)
c.SetLineStyle(3)
#c.Draw('PLsame')
c.Draw('same')
marker = list()
#Paint a specifical curve
for pair in paintCurves:
curves['roc'][pair.first].SetLineWidth(1)
curves['roc'][pair.first].SetLineStyle(1)
#curves['roc'][pair.first].SetMarkerStyle(7)
#curves['roc'][pair.first].SetMarkerColor(kBlue)
curves['roc'][pair.first].SetLineColor(pair.second)
#curves['roc'][pair.first].Draw('PLsame')
curves['roc'][pair.first].Draw('same')
if ref == 'SP':
faVec = curves['roc'][pair.first].GetX()
detVec = curves['roc'][pair.first].GetY()
from RingerCore import calcSP
spVec = [
calcSP(detVec[i], 1 - faVec[i])
for i in range(curves['roc'][pair.first].GetN())
]
imax = spVec.index(max(spVec))
from ROOT import TMarker
marker.append(TMarker(faVec[imax], detVec[imax], 4))
marker[-1].SetMarkerColor(pair.second)
marker[-1].Draw('same')
#Update Canvas
canvas.Modified()
canvas.Update()
canvas.SaveAs(savename)
del canvas
return savename
def plot_rocs(plotObjects, kwargs):
from ROOT import kCyan, kRed, kGreen, kBlue, kBlack, kMagenta, kGray, kWhite, kYellow
Colors = [kBlue, kRed, kMagenta, kBlack, kCyan, kGreen]
from RingerCore import StdPair as std_pair
from util import line, minmax
dset = kwargs["set"]
ref = kwargs["reference"]
refVal = kwargs["refVal"]
eps = kwargs["eps"]
savename = kwargs["cname"] + ".pdf"
# Some protection
if not ("op" in dset or "tst" in dset):
raise ValueError("Option set must be: tst (test) or val (validation)")
if not ("SP" in ref or "Pd" in ref or "Pf" in ref):
raise ValueError("Option reference must be: SP, Pd or Pf")
# Create dict to hold all list plots
curves = dict()
# list of dicts to dict of lists
for name in plotObjects.keys():
curves[name] = plotObjects.tolist(name)
paintIdx = kwargs["paintListIdx"] # [best, worst]
paintCurves = [
std_pair(plotObjects.index_correction(paintIdx[0]), kBlack),
std_pair(plotObjects.index_correction(paintIdx[1]), kRed),
]
curves["roc"] = curves["roc_" + dset]
# Start to build all ROOT objects
from ROOT import TCanvas, gROOT, kTRUE
gROOT.SetBatch(kTRUE)
canvas = TCanvas("canvas", "canvas", 1600, 1300)
x_limits = [0.00, 0.40]
y_limits = [0.6, 1.03]
# create dummy graph
dummy = curves["roc"][0]
dummy.SetTitle("Receive Operation Curve")
dummy.GetXaxis().SetTitle("False Alarm")
dummy.GetYaxis().SetTitle("Detection")
dummy.GetHistogram().SetAxisRange(y_limits[0], y_limits[1], "Y")
dummy.GetHistogram().SetAxisRange(x_limits[0], x_limits[1], "X")
dummy.Draw("AL")
corredor = None
target = None
from ROOT import TBox
if ref == "Pf":
corredor = TBox(refVal - eps, y_limits[0], refVal + eps, y_limits[1])
target = line(refVal, y_limits[0], refVal, y_limits[1], kBlack, 2, 1, "")
elif ref == "Pd":
corredor = TBox(x_limits[0], refVal - eps, x_limits[1], refVal + eps)
target = line(x_limits[0], refVal, x_limits[1], refVal, kBlack, 2, 1, "")
if ref != "SP":
corredor.SetFillColor(kYellow - 9)
corredor.Draw("same")
target.Draw("same")
canvas.Modified()
canvas.Update()
# Plot curves
for c in curves["roc"]:
c.SetLineColor(kGray + 1)
# c.SetMarkerStyle(7)
# c.SetMarkerColor(kBlue)
c.SetLineWidth(1)
c.SetLineStyle(3)
# c.Draw('PLsame')
c.Draw("same")
marker = list()
# Paint a specifical curve
for pair in paintCurves:
curves["roc"][pair.first].SetLineWidth(1)
curves["roc"][pair.first].SetLineStyle(1)
# curves['roc'][pair.first].SetMarkerStyle(7)
# curves['roc'][pair.first].SetMarkerColor(kBlue)
curves["roc"][pair.first].SetLineColor(pair.second)
# curves['roc'][pair.first].Draw('PLsame')
curves["roc"][pair.first].Draw("same")
if ref == "SP":
faVec = curves["roc"][pair.first].GetX()
detVec = curves["roc"][pair.first].GetY()
from RingerCore import calcSP
spVec = [calcSP(detVec[i], 1 - faVec[i]) for i in range(curves["roc"][pair.first].GetN())]
imax = spVec.index(max(spVec))
from ROOT import TMarker
marker.append(TMarker(faVec[imax], detVec[imax], 4))
marker[-1].SetMarkerColor(pair.second)
marker[-1].Draw("same")
# Update Canvas
canvas.Modified()
canvas.Update()
canvas.SaveAs(savename)
del canvas
return savename
def calcPerformance(anex, netBin, netaBin):
from RingerCore import calcSP
import numpy as np
signalPassRef = 0
signalPassBest = 0
signalTotal = 0
backgroundPassRef = 0
backgroundPassBest = 0
backgroundTotal = 0
backgroundPassTuning = {k: 0 for k in anex[0][0]['sortMaximuns'][0].keys()}
signalPassTuning = dict(backgroundPassTuning)
for et in xrange(netBin):
for eta in xrange(netaBin):
nsignal = anex[et][eta]['nsignal']
perf = anex[et][eta]['perf']
nbackground = anex[et][eta]['nbackground']
for k in signalPassTuning.keys():
signalPassTuning[
k] += anex[et][eta]['sortMaximuns'][0][k] * nsignal / 10.0
backgroundPassTuning[k] += anex[et][eta]['sortMaximuns'][1][
k] * nbackground / 10.0
signalPassRef += nsignal * perf['refBench']['det'] / 100.0
signalPassBest += nsignal * perf['bestNetBench']['det'] / 100.0
signalTotal += nsignal
backgroundPassRef += nbackground * perf['refBench']['fa'] / 100.0
backgroundPassBest += nbackground * perf['bestNetBench']['fa'] / 100
backgroundTotal += nbackground
#
#
pdArray = np.array(signalPassTuning.values()) / (signalTotal / 10.0) * 100
pfArray = np.array(
backgroundPassTuning.values()) / (backgroundTotal / 10.0) * 100
spList = [
calcSP(signalPassTuning[k] * 100 / (signalTotal / 10.0),
100 - backgroundPassTuning[k] * 100 / (backgroundTotal / 10.0))
for k in signalPassTuning.keys()
]
spMeanTuning = np.mean(spList)
spStdTuning = np.std(spList)
pdMeanTuning = pdArray.mean()
pdStdTuning = pdArray.std()
pfMeanTuning = pfArray.mean()
pfStdTuning = pfArray.std()
pdBest = signalPassBest / float(signalTotal) * 100
pdRef = signalPassRef / float(signalTotal) * 100
pfBest = backgroundPassBest / float(backgroundTotal) * 100
pfRef = backgroundPassRef / float(backgroundTotal) * 100
spBest = calcSP(pdBest, 100 - pfBest)
spRef = calcSP(pdRef, 100 - pfRef)
return {
'spTuning': spMeanTuning,
'spStdTuning': spStdTuning,
'pdTuning': pdMeanTuning,
'pdStdTuning': pdStdTuning,
'pfTuning': pfMeanTuning,
'pfStdTuning': pfStdTuning,
'pdBest': '%.2f' % (pdBest),
'pdRef': '%.2f' % (pdRef),
'pfBest': '%.2f' % (pfBest),
'pfRef': '%.2f' % (pfRef),
'spBest': '%.2f' % (spBest),
'spRef': '%.2f' % (spRef),
}