Ejemplo n.º 1
0
def find_threshold_best_sp(sgn_hist, noise_hist):

    from RingerCore import calcSP
    nbins = sgn_hist.GetNbinsX()
    det_total = sgn_hist.Integral(0, nbins + 1)
    noise_total = noise_hist.Integral(0, nbins + 1)

    sp_max = -999
    best_threshold = -999
    best_det = -999
    best_fa = -999
    i = 0
    while i < nbins + 1:
        # Detection
        det_passed = sgn_hist.Integral(i, nbins + 1)
        # False alarm
        noise_passed = noise_hist.Integral(i, nbins + 1)

        det = det_passed / det_total
        fa = noise_passed / noise_total
        sp = calcSP(det, 1 - fa)
        if sp > sp_max:
            sp_max = sp
            best_det = det
            best_fa = fa
            threshold = sgn_hist.GetBinCenter(i)
        i += 1

    return sp_max, best_det, best_fa, threshold
Ejemplo n.º 2
0
 def __init__(self, name, ref, tobj, opObj, bobj):
   
   MonitoringOperationInfo.__init__(self, opObj)
   #name
   self._name   = name
   self._perf = dict()
   self._ref = dict()
   #Retrieve information from benchmark
   self._ref['reference'] = bobj['reference']
   self._ref['det'] = bobj['signal_efficiency']['efficiency']
   self._ref['fa'] = bobj['background_efficiency']['efficiency']
   #Calculate SP from reference
   self._ref['sp'] = calcSP(self._ref['det'], 100-self._ref['fa'])
   #Hold values
   for key in self._keys:  self._perf[key] = tobj[key]*100
Ejemplo n.º 3
0
 def getEffPoint( self, name, subset = [None, None], outputs = [None, None], pileup = [None,None], thres = None, makeCorr = True ):
   from TuningTools.Neural import PerformancePoint
   auc = self.rawPerf.auc if self.rawPerf else -1
   if not isinstance(subset, (tuple,list)): 
     if subset is None: 
       if not(any([o is None for o in outputs])): self._fatal("Subset must be specified when outputs is used.")
       subset = self.subset
     subset = [CuratedSubset.tosgn(subset),CuratedSubset.tobkg(subset)]
   else:
     if len(subset) is 1:
       if subset[0] is None: 
         if not(any([o is None for o in outputs])): self._fatal("Subset must be specified when outputs is used.")
         subset = [self.subset]
       subset = [CuratedSubset.tosgn(subset[0]),CuratedSubset.tobkg(subset[0])]
     else:
       if any([s is None for s in subset]): 
         if not(any([o is None for o in outputs])): self._fatal("Subset must be specified when outputs is used.")
         subset = [self.subset, self.subset]
       subset = [CuratedSubset.tosgn(subset[0]),CuratedSubset.tobkg(subset[1])]
   self._effSubset = subset
   if any([o is None for o in outputs]):
     #if outputs[0] is None:
     if isinstance(subset, (list,tuple)):
       # FIXME This assumes that sgnOut is cached:
       outputs = [(self.getOutput(CuratedSubset.topattern(s)) if CuratedSubset.tobinary(s) is not self.subset else o )
                  for o, s in zip([self.sgnOut,self.bkgOut], subset)]
     else:
       if CuratedSubset.tobinary(subset) is not self.subset:
         outputs = self.getOutput(CuratedSubset.topattern(subset))
       else:
         outputs = [self.sgnOut,self.bkgOut]
     # NOTE: This can be commented out to improve speed
     try:
       from libTuningToolsLib import genRoc
     except ImportError:
       from libTuningTools import genRoc
     o = genRoc(outputs[0], outputs[1], +self._ol, -self._ol, 0.001 )
     auc = Roc( o ).auc
   self._effOutput = outputs
   if thres is None: thres = self.thres if makeCorr else self.rawThres
   pd = self._calcEff( subset[0], output = outputs[0], pileup = pileup[0], thres = thres, makeCorr = makeCorr )
   pf = self._calcEff( subset[1], output = outputs[1], pileup = pileup[1], thres = thres, makeCorr = makeCorr )
   sp = calcSP(pd, 1. - pf)
   return PerformancePoint( name, sp, pd, pf, thres, perc = False, auc = auc
                          , etBinIdx = self.etBinIdx, etaBinIdx = self.etaBinIdx
                          , etBin = self.etBin, etaBin = self.etaBin )
Ejemplo n.º 4
0
    def __init__(self, name, ref, tobj, opObj, bobj):

        MonitoringOperationInfo.__init__(self, opObj)
        #name
        self._name = name
        self._perf = dict()
        self._ref = dict()
        #Retrieve information from benchmark
        self._ref['reference'] = bobj['reference']
        try:
            self._ref['det'] = bobj['signalEfficiency']['efficiency']
            self._ref['fa'] = bobj['backgroundEfficiency']['efficiency']
        except:
            self._ref['det'] = bobj['signal_efficiency']['efficiency']
            self._ref['fa'] = bobj['background_efficiency']['efficiency']
        #Calculate SP from reference
        self._ref['sp'] = calcSP(self._ref['det'], 100 - self._ref['fa'])
        #Hold values
        for key in self._keys:
            self._perf[key] = tobj[key] * 100
Ejemplo n.º 5
0
def plot_rocs(plotObjects, kwargs):

    from ROOT import kCyan, kRed, kGreen, kBlue, kBlack, kMagenta, kGray, kWhite, kYellow
    Colors = [kBlue, kRed, kMagenta, kBlack, kCyan, kGreen]
    from RingerCore import StdPair as std_pair
    from util import line, minmax

    dset = kwargs['set']
    ref = kwargs['reference']
    refVal = kwargs['refVal']
    eps = kwargs['eps']
    savename = kwargs['cname'] + '.pdf'

    #Some protection
    if not ('operation' in dset or 'tst' in dset):
        raise ValueError('Option set must be: tst (test) or val (validation)')
    if not ('SP' in ref or 'Pd' in ref or 'Pf' in ref):
        raise ValueError('Option reference must be: SP, Pd or Pf')

    #Create dict to hold all list plots
    curves = dict()
    #list of dicts to dict of lists
    for name in plotObjects.keys():
        curves[name] = plotObjects.tolist(name)

    paintIdx = kwargs['paintListIdx']  # [best, worst]
    paintCurves = [
        std_pair(plotObjects.index_correction(paintIdx[0]), kBlack),
        std_pair(plotObjects.index_correction(paintIdx[1]), kRed)
    ]
    curves['roc'] = curves['roc_' + dset]

    #Start to build all ROOT objects
    from ROOT import TCanvas, gROOT, kTRUE
    gROOT.SetBatch(kTRUE)
    canvas = TCanvas('canvas', 'canvas', 1600, 1300)

    x_limits = [0.00, 0.40]
    y_limits = [0.6, 1.03]

    #create dummy graph
    dummy = curves['roc'][0]
    dummy.SetTitle('Receive Operation Curve')
    dummy.GetXaxis().SetTitle('False Alarm')
    dummy.GetYaxis().SetTitle('Detection')
    dummy.GetHistogram().SetAxisRange(y_limits[0], y_limits[1], 'Y')
    dummy.GetHistogram().SetAxisRange(x_limits[0], x_limits[1], 'X')
    dummy.Draw('AL')

    corredor = None
    target = None
    from ROOT import TBox
    if ref == 'Pf':
        corredor = TBox(refVal - eps, y_limits[0], refVal + eps, y_limits[1])
        target = line(refVal, y_limits[0], refVal, y_limits[1], kBlack, 2, 1,
                      '')
    elif ref == 'Pd':
        corredor = TBox(x_limits[0], refVal - eps, x_limits[1], refVal + eps)
        target = line(x_limits[0], refVal, x_limits[1], refVal, kBlack, 2, 1,
                      '')

    if ref != 'SP':
        corredor.SetFillColor(kYellow - 9)
        corredor.Draw('same')
        target.Draw('same')
        canvas.Modified()
        canvas.Update()

    #Plot curves
    for c in curves['roc']:
        c.SetLineColor(kGray + 1)
        #c.SetMarkerStyle(7)
        #c.SetMarkerColor(kBlue)
        c.SetLineWidth(1)
        c.SetLineStyle(3)
        #c.Draw('PLsame')
        c.Draw('same')

    marker = list()
    #Paint a specifical curve
    for pair in paintCurves:
        curves['roc'][pair.first].SetLineWidth(1)
        curves['roc'][pair.first].SetLineStyle(1)
        #curves['roc'][pair.first].SetMarkerStyle(7)
        #curves['roc'][pair.first].SetMarkerColor(kBlue)
        curves['roc'][pair.first].SetLineColor(pair.second)
        #curves['roc'][pair.first].Draw('PLsame')
        curves['roc'][pair.first].Draw('same')

        if ref == 'SP':
            faVec = curves['roc'][pair.first].GetX()
            detVec = curves['roc'][pair.first].GetY()
            from RingerCore import calcSP
            spVec = [
                calcSP(detVec[i], 1 - faVec[i])
                for i in range(curves['roc'][pair.first].GetN())
            ]
            imax = spVec.index(max(spVec))
            from ROOT import TMarker
            marker.append(TMarker(faVec[imax], detVec[imax], 4))
            marker[-1].SetMarkerColor(pair.second)
            marker[-1].Draw('same')

    #Update Canvas
    canvas.Modified()
    canvas.Update()
    canvas.SaveAs(savename)
    del canvas

    return savename
Ejemplo n.º 6
0
def plot_rocs(plotObjects, kwargs):

    from ROOT import kCyan, kRed, kGreen, kBlue, kBlack, kMagenta, kGray, kWhite, kYellow

    Colors = [kBlue, kRed, kMagenta, kBlack, kCyan, kGreen]
    from RingerCore import StdPair as std_pair
    from util import line, minmax

    dset = kwargs["set"]
    ref = kwargs["reference"]
    refVal = kwargs["refVal"]
    eps = kwargs["eps"]
    savename = kwargs["cname"] + ".pdf"

    # Some protection
    if not ("op" in dset or "tst" in dset):
        raise ValueError("Option set must be: tst (test) or val (validation)")
    if not ("SP" in ref or "Pd" in ref or "Pf" in ref):
        raise ValueError("Option reference must be: SP, Pd or Pf")

    # Create dict to hold all list plots
    curves = dict()
    # list of dicts to dict of lists
    for name in plotObjects.keys():
        curves[name] = plotObjects.tolist(name)

    paintIdx = kwargs["paintListIdx"]  # [best, worst]
    paintCurves = [
        std_pair(plotObjects.index_correction(paintIdx[0]), kBlack),
        std_pair(plotObjects.index_correction(paintIdx[1]), kRed),
    ]
    curves["roc"] = curves["roc_" + dset]

    # Start to build all ROOT objects
    from ROOT import TCanvas, gROOT, kTRUE

    gROOT.SetBatch(kTRUE)
    canvas = TCanvas("canvas", "canvas", 1600, 1300)

    x_limits = [0.00, 0.40]
    y_limits = [0.6, 1.03]

    # create dummy graph
    dummy = curves["roc"][0]
    dummy.SetTitle("Receive Operation Curve")
    dummy.GetXaxis().SetTitle("False Alarm")
    dummy.GetYaxis().SetTitle("Detection")
    dummy.GetHistogram().SetAxisRange(y_limits[0], y_limits[1], "Y")
    dummy.GetHistogram().SetAxisRange(x_limits[0], x_limits[1], "X")
    dummy.Draw("AL")

    corredor = None
    target = None
    from ROOT import TBox

    if ref == "Pf":
        corredor = TBox(refVal - eps, y_limits[0], refVal + eps, y_limits[1])
        target = line(refVal, y_limits[0], refVal, y_limits[1], kBlack, 2, 1, "")
    elif ref == "Pd":
        corredor = TBox(x_limits[0], refVal - eps, x_limits[1], refVal + eps)
        target = line(x_limits[0], refVal, x_limits[1], refVal, kBlack, 2, 1, "")

    if ref != "SP":
        corredor.SetFillColor(kYellow - 9)
        corredor.Draw("same")
        target.Draw("same")
        canvas.Modified()
        canvas.Update()

    # Plot curves
    for c in curves["roc"]:
        c.SetLineColor(kGray + 1)
        # c.SetMarkerStyle(7)
        # c.SetMarkerColor(kBlue)
        c.SetLineWidth(1)
        c.SetLineStyle(3)
        # c.Draw('PLsame')
        c.Draw("same")

    marker = list()
    # Paint a specifical curve
    for pair in paintCurves:
        curves["roc"][pair.first].SetLineWidth(1)
        curves["roc"][pair.first].SetLineStyle(1)
        # curves['roc'][pair.first].SetMarkerStyle(7)
        # curves['roc'][pair.first].SetMarkerColor(kBlue)
        curves["roc"][pair.first].SetLineColor(pair.second)
        # curves['roc'][pair.first].Draw('PLsame')
        curves["roc"][pair.first].Draw("same")

        if ref == "SP":
            faVec = curves["roc"][pair.first].GetX()
            detVec = curves["roc"][pair.first].GetY()
            from RingerCore import calcSP

            spVec = [calcSP(detVec[i], 1 - faVec[i]) for i in range(curves["roc"][pair.first].GetN())]
            imax = spVec.index(max(spVec))
            from ROOT import TMarker

            marker.append(TMarker(faVec[imax], detVec[imax], 4))
            marker[-1].SetMarkerColor(pair.second)
            marker[-1].Draw("same")

    # Update Canvas
    canvas.Modified()
    canvas.Update()
    canvas.SaveAs(savename)
    del canvas

    return savename
Ejemplo n.º 7
0
def calcPerformance(anex, netBin, netaBin):
    from RingerCore import calcSP
    import numpy as np
    signalPassRef = 0
    signalPassBest = 0
    signalTotal = 0
    backgroundPassRef = 0
    backgroundPassBest = 0
    backgroundTotal = 0

    backgroundPassTuning = {k: 0 for k in anex[0][0]['sortMaximuns'][0].keys()}
    signalPassTuning = dict(backgroundPassTuning)
    for et in xrange(netBin):
        for eta in xrange(netaBin):
            nsignal = anex[et][eta]['nsignal']
            perf = anex[et][eta]['perf']
            nbackground = anex[et][eta]['nbackground']
            for k in signalPassTuning.keys():
                signalPassTuning[
                    k] += anex[et][eta]['sortMaximuns'][0][k] * nsignal / 10.0
                backgroundPassTuning[k] += anex[et][eta]['sortMaximuns'][1][
                    k] * nbackground / 10.0
            signalPassRef += nsignal * perf['refBench']['det'] / 100.0
            signalPassBest += nsignal * perf['bestNetBench']['det'] / 100.0
            signalTotal += nsignal
            backgroundPassRef += nbackground * perf['refBench']['fa'] / 100.0
            backgroundPassBest += nbackground * perf['bestNetBench']['fa'] / 100
            backgroundTotal += nbackground
            #
        #
    pdArray = np.array(signalPassTuning.values()) / (signalTotal / 10.0) * 100
    pfArray = np.array(
        backgroundPassTuning.values()) / (backgroundTotal / 10.0) * 100
    spList = [
        calcSP(signalPassTuning[k] * 100 / (signalTotal / 10.0),
               100 - backgroundPassTuning[k] * 100 / (backgroundTotal / 10.0))
        for k in signalPassTuning.keys()
    ]
    spMeanTuning = np.mean(spList)
    spStdTuning = np.std(spList)

    pdMeanTuning = pdArray.mean()
    pdStdTuning = pdArray.std()

    pfMeanTuning = pfArray.mean()
    pfStdTuning = pfArray.std()

    pdBest = signalPassBest / float(signalTotal) * 100
    pdRef = signalPassRef / float(signalTotal) * 100

    pfBest = backgroundPassBest / float(backgroundTotal) * 100
    pfRef = backgroundPassRef / float(backgroundTotal) * 100

    spBest = calcSP(pdBest, 100 - pfBest)
    spRef = calcSP(pdRef, 100 - pfRef)

    return {
        'spTuning': spMeanTuning,
        'spStdTuning': spStdTuning,
        'pdTuning': pdMeanTuning,
        'pdStdTuning': pdStdTuning,
        'pfTuning': pfMeanTuning,
        'pfStdTuning': pfStdTuning,
        'pdBest': '%.2f' % (pdBest),
        'pdRef': '%.2f' % (pdRef),
        'pfBest': '%.2f' % (pfBest),
        'pfRef': '%.2f' % (pfRef),
        'spBest': '%.2f' % (spBest),
        'spRef': '%.2f' % (spRef),
    }