def ttbarShapeSysSR(topfileName="hist_ttbar.root",
                    distributionName= "mHH_l",
                    signal_region  = "33",
                    compare_region = "44",
                    btag_WP        = "77",
                    smoothing_func = "Exp",
                    SmoothRange = (1200, 3000),# (100, 2500),
                    makePlots = False,
                    verbose = False,
                    outfileNameBase="TopShapeSRSysfitSmooth.root"):
    
    topfile  = R.TFile(topfileName,"READ")

    ttbarShapeSRSyst_Dict= {}

    colorlist = [ R.kGreen, R.kOrange, R.kMagenta, R.kCyan, R.kPink, (R.kAzure+1), R.kGreen+2, R.kOrange+5]        

    
    ## get top SR shape
    folder_sig = HistLocStr(distributionName, signal_region[0], signal_region[1], btag_WP, "SB")  #folder( r[0], r[1], btag_WP)
    top_sig    = topfile.Get(folder_sig).Clone("top_sig_"+signal_region)
    top_sig.SetDirectory(0)
    top_sig.Rebin(10)


    ## get top comparison shape
    folder_comp = HistLocStr(distributionName, compare_region[0], compare_region[1], btag_WP, "SB")  #folder( r[0], r[1], btag_WP)
    top_comp    = topfile.Get(folder_comp).Clone("top_comp_"+compare_region)
    top_comp.SetDirectory(0)
    top_comp.Rebin(10)

    ## remove negative values
    ## assume same binning, else division won't work later
    for ibin in range(1, top_sig.GetNbinsX()+1):
        if top_sig.GetBinContent(ibin) < 0:
            top_sig.SetBinContent(ibin, 0)
            top_sig.SetBinError(ibin, 0)
            
        if top_comp.GetBinContent(ibin) < 0:
            top_comp.SetBinContent(ibin, 0)
            top_comp.SetBinError(ibin, 0)

    ## normalize to same area
    low_x = top_comp.GetXaxis().FindBin(SmoothRange[0])
    low_y = top_comp.GetXaxis().FindBin(SmoothRange[1])
    top_sig.Scale( top_comp.Integral(low_x, low_y) / top_sig.Integral(low_x, low_y) )

    c=R.TCanvas("c1_topsys","c1_topsys")
    xleg, yleg = 0.52, 0.7
    leg = R.TLegend(xleg, yleg, xleg+0.3, yleg+0.2)
    leg.SetFillColor(0)
    leg.SetBorderSize(0)
    leg.SetMargin(0.3)
    top_comp.SetXTitle("m_{JJ} [GeV]")
    top_comp.GetXaxis().SetRangeUser(0, 4000)
    top_comp.SetYTitle("Entries")
    top_comp.Draw("E1")
    leg.AddEntry(top_comp, "Top Comparison Distribution", "LP")

    #################################
    ## smooth bkg and data
    ##################################
    top_comp_sm   = smoothfit.smoothfit(top_comp, fitFunction = smoothing_func, fitRange = SmoothRange, makePlots = False, verbose = False, outfileName="top_comp_smoothfit_TopShape4b.root")
    top_comp_sm_h = smoothfit.MakeSmoothHisto(top_comp, top_comp_sm["nom"])

    top_comp_sm["nom"].SetLineColor(R.kBlack)
    top_comp_sm["nom"].Draw("same")
    leg.AddEntry(top_comp_sm["nom"], "Top Comparison Distribution Smooth", "L")

    top_sig_sm = smoothfit.smoothfit(top_sig, fitFunction = smoothing_func, fitRange = SmoothRange, makePlots = False, verbose = False, outfileName="top_sig_smoothfit_TopShape4.root")
    top_sig_sm_h = smoothfit.MakeSmoothHisto(top_sig, top_sig_sm["nom"])

    top_sig_sm["nom"].SetLineColor(R.kBlue)
    top_sig_sm["nom"].Draw("same")
    leg.AddEntry(top_sig_sm["nom"], "Top Nominal Distribution Smooth", "L")


    rfunc1 = top_comp_sm["nom"]
    rfunc2 = top_sig_sm["nom"]
    def rfunc_ratio(x):
        return rfunc1.Eval(x[0]) / rfunc2.Eval(x[0])
    xMax = top_comp.GetXaxis().GetBinUpEdge(top_comp.GetXaxis().GetNbins())
    ratio_sm = R.TF1("ratio_topsys_sm", rfunc_ratio, SmoothRange[0], xMax, 0)

    for ivar in range(len(top_comp_sm["vars"])):
        dup = top_comp_sm["vars"][ivar][0]
        ddw = top_comp_sm["vars"][ivar][1]

        dup.SetLineColor(colorlist[ivar])
        ddw.SetLineColor(colorlist[ivar])

        dup.Draw("same")
        ddw.Draw("same")
        leg.AddEntry(dup, "Top Comparison Smooth Variation", "L")


        top_comp_r_qup = smoothfit.MakeSmoothHisto(top_comp, dup)
        top_comp_r_qdw = smoothfit.MakeSmoothHisto(top_comp, ddw)

        for ibin in range(1,  top_comp_sm_h.GetNbinsX()+1):
            err_val = np.max( np.abs( [ top_comp_sm_h.GetBinContent(ibin) - top_comp_r_qup.GetBinContent(ibin), top_comp_sm_h.GetBinContent(ibin) - top_comp_r_qdw.GetBinContent(ibin)] ) )
            top_comp_sm_h.SetBinError(ibin, np.sqrt( top_comp_sm_h.GetBinError(ibin)**2 + err_val**2) )

    c.SetLogy(1)
    leg.Draw("same")
    c.SaveAs(outfileNameBase.split(".root")[0] + "_sig"+signal_region+"_comp"+ compare_region + ".root")
    c.SaveAs(outfileNameBase.split(".root")[0] + "_sig"+signal_region+"_comp"+ compare_region + ".pdf")
    c.Close()


    h_ratio_cr_nom = top_comp_sm_h.Clone("top_comp_sm_h_TopShape4b")
    h_ratio_cr_nom.Divide( top_comp_sm["nom"] )
    h_ratio_cr_nom.SetDirectory(0)
            
    h_ratio_cr = top_comp_sm["nom"].GetHistogram()
    h_ratio_cr.Divide( top_sig_sm["nom"] )
    h_ratio_cr.SetDirectory(0)
        

    ttbarShapeSRSyst_Dict["Shape"] = ratio_sm
        
        
    c2=R.TCanvas("c2_topsys","c2_topsys")
    leg = R.TLegend(0.2,0.7,0.5,0.9)
    leg.SetFillColor(0)
    h_ratio_cr_nom.SetFillColor(R.kBlack)
    h_ratio_cr_nom.SetFillStyle(3004)
    h_ratio_cr_nom.SetMarkerSize(0)
    h_ratio_cr_nom.GetXaxis().SetRangeUser(1000, 4000)
    h_ratio_cr_nom.GetYaxis().SetRangeUser(0, 3)
    h_ratio_cr_nom.GetXaxis().SetLabelSize(0.04)
    h_ratio_cr_nom.GetYaxis().SetLabelSize(0.04)
    h_ratio_cr_nom.SetXTitle("m_{JJ} [GeV]")
    h_ratio_cr_nom.SetYTitle("Ratio")
    h_ratio_cr_nom.Draw("E2")
    leg.AddEntry(h_ratio_cr_nom, "Nominal", "LF")


    h_ratio_cr.SetLineColor( R.kBlue )
    h_ratio_cr.Draw("same")
    leg.AddEntry(h_ratio_cr, "Predicted", "L")


    #ratio_sm.Draw("same")

    leg.Draw("same")
    c2.SaveAs(outfileNameBase.split(".root")[0] + "_sig"+signal_region+"_comp"+ compare_region +"_ratio.root")
    c2.SaveAs(outfileNameBase.split(".root")[0] + "_sig"+signal_region+"_comp"+ compare_region +"_ratio.pdf")
    c2.Close()

    topfile.Close()

    return ttbarShapeSRSyst_Dict
def smoothFuncCompare(
    histo,
    fitFunction="ExpModGauss",
    fitRange=(100, 3000),
    funcCompareRange=(900, 3000),
    makePlots=False,
    plotExtra=True,
    verbose=False,
    outfileName="ExpModGaussSmoothFuncCompare.root",
):

    colorlist = [R.kBlue, R.kGreen, R.kOrange, R.kMagenta, R.kCyan, R.kPink, (R.kAzure + 1), R.kGreen + 2]

    namestr = outfileName.split(".root")[0]

    h_clone = histo.Clone()
    h_clone.SetDirectory(0)

    nominal_result = smoothfit.smoothfit(
        h_clone,
        fitFunction=fitFunction,
        fitRange=fitRange,
        makePlots=False,
        verbose=verbose,
        outfileName=fitFunction + "_" + outfileName,
    )

    nominal_hist = smoothfit.MakeSmoothHisto(h_clone, nominal_result["nom"])

    results = {}
    results_hist = {}
    results_hist_ud = {}

    for theFunc in ["Exp", "MJ2", "MJ3", "MJ4", "MJ5", "MJ6", "MJ7", "MJ8"]:
        results[theFunc] = smoothfit.smoothfit(
            h_clone,
            fitFunction=theFunc,
            fitRange=funcCompareRange,
            makePlots=False,
            verbose=verbose,
            outfileName=theFunc + "_" + outfileName,
        )
        results_hist[theFunc] = smoothfit.MakeSmoothHisto(h_clone, results[theFunc]["nom"])

        results_hist_ud[theFunc] = {}
        for ivar in range(len(results[theFunc]["vars"])):
            results_hist_ud[theFunc]["up" + str(ivar)] = smoothfit.MakeSmoothHisto(
                h_clone, results[theFunc]["vars"][ivar][0]
            )
            results_hist_ud[theFunc]["dw" + str(ivar)] = smoothfit.MakeSmoothHisto(
                h_clone, results[theFunc]["vars"][ivar][1]
            )

    histo_up = nominal_hist.Clone(histo.GetName() + "_" + namestr + "_up")
    histo_up.SetDirectory(0)
    histo_dw = nominal_hist.Clone(histo.GetName() + "_" + namestr + "_dw")
    histo_dw.SetDirectory(0)

    histo_up_super = nominal_hist.Clone(histo.GetName() + "_" + namestr + "_up_super")
    histo_up_super.SetDirectory(0)
    histo_dw_super = nominal_hist.Clone(histo.GetName() + "_" + namestr + "_dw_super")
    histo_dw_super.SetDirectory(0)

    for ibin in range(1, histo.GetNbinsX() + 1):
        if histo.GetBinLowEdge(ibin) + histo.GetBinWidth(ibin) < funcCompareRange[0]:
            continue

        deltas = []
        deltas_super = []
        for theFunc in ["Exp", "MJ2", "MJ3", "MJ4", "MJ5", "MJ6", "MJ7", "MJ8"]:
            deltas.append(np.abs(nominal_hist.GetBinContent(ibin) - results_hist[theFunc].GetBinContent(ibin)))

            for ivarh in results_hist_ud[theFunc]:
                deltas_super.append(
                    np.abs(nominal_hist.GetBinContent(ibin) - results_hist_ud[theFunc][ivarh].GetBinContent(ibin))
                )

        theDelta = np.max(deltas)
        theDelta_super = np.max(deltas_super)
        histo_up.SetBinContent(ibin, histo_up.GetBinContent(ibin) + theDelta)
        histo_dw.SetBinContent(ibin, histo_dw.GetBinContent(ibin) - theDelta)

        histo_up_super.SetBinContent(ibin, histo_up.GetBinContent(ibin) + theDelta_super)
        histo_dw_super.SetBinContent(ibin, histo_dw.GetBinContent(ibin) - theDelta_super)

    smoothFuncCompSyst = {"up": histo_up, "dw": histo_dw, "up_super": histo_up_super, "dw_super": histo_dw_super}

    if makePlots:
        f = R.TFile(outfileName, "RECREATE")

        c = R.TCanvas("c1", "c1")
        # R.SetOwnership(c,False)
        leg = R.TLegend(0.1, 0.7, 0.48, 0.9)
        leg.SetFillColor(0)

        h_clone.SetLineColor(R.kBlack)
        h_clone.Draw()
        leg.AddEntry(histo, "Histogram", "L")

        icol = 0
        ivar0 = True
        err_hist_ratio = None

        err_hist = smoothfit.MakeSmoothHisto(histo, nominal_result["nom"])
        err_hist.SetDirectory(0)

        for ivar in range(len(nominal_result["vars"])):
            err_hist_up = smoothfit.MakeSmoothHisto(histo, nominal_result["vars"][ivar][0])
            err_hist_dw = smoothfit.MakeSmoothHisto(histo, nominal_result["vars"][ivar][1])

            for ibin in range(1, err_hist.GetNbinsX() + 1):
                err_val = np.max(
                    np.abs(
                        [
                            err_hist.GetBinContent(ibin) - err_hist_up.GetBinContent(ibin),
                            err_hist.GetBinContent(ibin) - err_hist_dw.GetBinContent(ibin),
                        ]
                    )
                )
                err_hist.SetBinError(ibin, np.sqrt(err_hist.GetBinError(ibin) ** 2 + err_val ** 2))

        err_hist_ratio = err_hist.Clone("err_hist_ratio__" + namestr)
        err_hist_ratio.SetDirectory(0)
        err_hist_ratio.Divide(nominal_result["nom"])

        err_hist.SetFillColor(R.kBlack)
        err_hist.SetFillStyle(3001)
        err_hist.Draw("sameE3")
        leg.AddEntry(err_hist, "smoothing error", "F")

        for theFunc in ["Exp", "MJ2", "MJ3", "MJ4", "MJ5", "MJ6", "MJ7", "MJ8"]:

            # print results[theFunc]["nom"], results[theFunc]["nom"].Eval(1000), results[theFunc]["nom"].Eval(2000), results[theFunc]["nom"].Eval(3000)

            results[theFunc]["nom"].SetLineColor(colorlist[icol])
            results[theFunc]["nom"].Draw("same")
            leg.AddEntry(results[theFunc]["nom"], theFunc, "L")

            if plotExtra:
                for ivar in range(len(results[theFunc]["vars"])):
                    results[theFunc]["vars"][ivar][0].SetLineColor(R.kGray + 2)
                    results[theFunc]["vars"][ivar][0].Draw("same")
                    results[theFunc]["vars"][ivar][1].SetLineColor(R.kGray + 2)
                    results[theFunc]["vars"][ivar][1].Draw("same")

            icol += 1

        if plotExtra:
            results_hist_ud["MJ2"]["up0"].SetLineColor(R.kGray + 2)
            leg.AddEntry(results_hist_ud["MJ2"]["up0"], "Param Variations", "L")
        leg.Draw()

        c2 = R.TCanvas("c2", "c2")
        # R.SetOwnership(c,False)
        print "err_hist_ratio", err_hist_ratio
        err_hist_ratio.SetFillColor(R.kBlack)
        err_hist_ratio.SetFillStyle(3001)
        err_hist_ratio.Draw("E2")

        icol = 0
        f_ratio = {}

        delta_ratio_super = {}
        for theFunc in ["Exp", "MJ2", "MJ3", "MJ4", "MJ5", "MJ6", "MJ7", "MJ8"]:

            h_ratio = results[theFunc]["nom"].GetHistogram()
            h_ratio.Divide(nominal_result["nom"])
            h_ratio.SetDirectory(0)

            h_ratio.SetLineColor(colorlist[icol])
            h_ratio.Draw("same")

            if plotExtra:
                for ivar in range(len(results[theFunc]["vars"])):
                    h_ratio_ud = results[theFunc]["vars"][ivar][0].GetHistogram()
                    h_ratio_ud.Divide(nominal_result["nom"])
                    h_ratio_ud.SetDirectory(0)
                    h_ratio_ud.SetLineColor(R.kGray + 2)
                    h_ratio_ud.Draw("same")

                    delta_ratio_super[theFunc + "_" + str(ivar) + "_up"] = h_ratio_ud.GetBinContent(
                        h_ratio_ud.FindBin(3000)
                    )

                    h_ratio_ud = results[theFunc]["vars"][ivar][1].GetHistogram()
                    h_ratio_ud.Divide(nominal_result["nom"])
                    h_ratio_ud.SetDirectory(0)
                    h_ratio_ud.SetLineColor(R.kGray + 2)
                    h_ratio_ud.Draw("same")

                    delta_ratio_super[theFunc + "_" + str(ivar) + "_dw"] = h_ratio_ud.GetBinContent(
                        h_ratio_ud.FindBin(3000)
                    )

            # print f_copy, f_ratio[theFunc], f_ratio[theFunc].Eval(1000), f_ratio[theFunc].Eval(2000), f_ratio[theFunc].Eval(3000)
            icol += 1

        leg.Draw()
        # for drs in delta_ratio_super:
        #    print drs, delta_ratio_super[drs]

        f.WriteTObject(c)
        f.WriteTObject(c2)
        f.Close()

    return smoothFuncCompSyst
예제 #3
0
def HistoAnalysis(datafileName="hist_data.root",
                  topfileName="hist_ttbar.root",
                  zjetfileName="hist_Zjets.root",
                  distributionName= "DiJetMass",
                  n_trkjet  = ["4","4"],
                  n_btag    = ["4","3"],
                  btag_WP     = "77",
                  NRebin = 1,
                  use_one_top_nuis = False,
                  use_scale_top_2b = False,
                  nbtag_top_shape_normFit = None,
                  nbtag_top_shape_SRPred = "3",
                  rebinFinal = None,
                  smoothing_func = "Exp",
                  inputFitResult = None,
                  inputQCDSyst_Dict = None,
                  doSmoothing = True,
                  addSmoothErrorBin = False,
                  qcdSmoothRange = (900, 2000),# (100, 2500),
                  topSmoothRange = (850, 2000), #(100, 2000),
                  isSystematicVariation = False,
                  verbose = False,
                  makeOutputFiles = True,
                  MassRegionName = "SR"):

    global func1
    global func2
    
    ##### Parse Inputs ############################################
    dist_name   = distributionName
    
    num_trkjet  = np.asarray(n_trkjet)
    if num_trkjet.shape==():
        num_trkjet = np.asarray([n_trkjet])

    num_btag    = np.asarray(n_btag)
    if num_btag.shape==():
        num_btag = np.asarray([n_btag])
    if num_btag.shape!=num_trkjet.shape:
        print "Must have same number of track jet and b-tag regions specified"
        sys.exit(0)
  
    btag_WP     = btag_WP
    
    n_rebin     = NRebin

    nbtag_top_shape = nbtag_top_shape_SRPred
    topShape_nbtag = nbtag_top_shape
    if nbtag_top_shape == None:
        topShape_nbtag = num_btag

    useOneTopNuis = use_one_top_nuis

    scaleTop2b = use_scale_top_2b

    n_channels = num_trkjet.shape[0]

    regions = [ num_trkjet[i]+num_btag[i] for i in range(n_channels) ]


    ##for outputing
    do_smoothing  = (doSmoothing if (distributionName=="DiJetMass" or distributionName=="DiJetMassPrime") else False)   # qi

    ##################################################################



    
    ##### Storage Variables ############################################
    output_Dict = { }

    
    Nbkg_dict    = {  }
    Nbkg_SysList = {  }
    for ir in regions:
        Nbkg_dict[ir]    = { "qcd":0,  "top":0,  "zjet":0,  "bkg":0 }
        Nbkg_SysList[ir] = { "qcd":[], "top":[], "zjet":[], "bkg":[] }


    vartxt = ''
    ##################################################################


    
    ##### Do Background Fits ############################################
    if inputFitResult == None:
        bkgFitResults = BkgFit. BackgroundFit(datafileName=datafileName,
                                          topfileName=topfileName,
                                          zjetfileName=zjetfileName,
                                          distributionName= "LeadCaloJetM",
                                          n_trkjet  = n_trkjet,
                                          n_btag    = n_btag,
                                          btag_WP     = btag_WP,
                                          NRebin = NRebin,
                                          use_one_top_nuis = use_one_top_nuis,
                                          use_scale_top_2b = use_scale_top_2b,
                                          nbtag_top_shape = nbtag_top_shape_normFit,
                                          makePlots = True,
                                          verbose = verbose )

    else:
        bkgFitResults = inputFitResult

    pvars = bkgFitResults["pvars"]

    output_Dict["fitResults"] = bkgFitResults
    ##################################################################


    ##### Get QCD Shape Systematics from CR  ##############################
    if MassRegionName == "SR":
        # should only affect SR
        if inputQCDSyst_Dict == None and (distributionName=="DiJetMass" or distributionName=="DiJetMassPrime"):  # qi
            QCDSyst_Dict = SystTools.QCDSystematics(datafileName=datafileName,
                                                    topfileName=topfileName,
                                                    zjetfileName=zjetfileName,
                                                    distributionName= dist_name,
                                                    n_trkjet  = n_trkjet,
                                                    n_btag    = n_btag,
                                                    btag_WP     = btag_WP,
                                                    mu_qcd_vals = bkgFitResults["muqcd"],
                                                    topscale_vals = bkgFitResults["topscale"],
                                                    NRebin = NRebin,
                                                    use_one_top_nuis = use_one_top_nuis,
                                                    use_scale_top_2b = use_scale_top_2b,
                                                    makePlots = True,
                                                    verbose = False, # True, # hack
                                                    outfileNameBase="QCDSysfit.root")
        elif inputQCDSyst_Dict != None:
            QCDSyst_Dict = inputQCDSyst_Dict

        else:
            QCDSyst_Dict = None
    else:
        QCDSyst_Dict = None

    output_Dict["QCDSystCR"] = QCDSyst_Dict
    ##################################################################



    

    ##### Get Signal Region Histograms ################################
    datafile = R.TFile(datafileName,"READ")
    topfile  = R.TFile(topfileName,"READ")
    zjetfile  = R.TFile(zjetfileName,"READ")


    histos = {}
    
    # collect all histograms
    for r in ["44","43","42","33","32"]:
        # folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP, "SR")  #folder( r[0], r[1], btag_WP)
        folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP, MassRegionName)  #folder( r[0], r[1], btag_WP)
        
        data_r   = datafile.Get(folder_r).Clone("data_"+r)
        data_r.SetDirectory(0)
        
        top_r    = topfile.Get(folder_r).Clone("top_"+r)
        top_r.SetDirectory(0)

        zjet_r   = CheckAndGet(zjetfile, folder_r, top_r).Clone("zjet_"+r)
        zjet_r.SetDirectory(0)

        for ibin in range(1, top_r.GetNbinsX()+1):
            if top_r.GetBinContent(ibin) < 0:
                top_r.SetBinContent(ibin, 0)
                top_r.SetBinError(ibin, 0)
                

        histos[r]     = {"data": data_r,  "top": top_r,  "zjet":zjet_r}

    datafile.Close()
    topfile.Close()
    ##################################################################



    
    ##### scaling and subtractions #################################
    for ir in range(len(regions)):
        r = regions[ir]

        output_Dict[r] = {"qcd":{}, "ttbar":{}, "zjet":{}}
        
        if makeOutputFiles:
            outfileStat = R.TFile("outfile_boosted_"+r+".root","RECREATE")
        
        r_2b = r[0]+"2"
        r_3b = r[0]+"3"

        top_2b = histos[r_2b]["top"].Clone("top_2b__"+r)
        if scaleTop2b:
            top_2b.Scale( (bkgFitResults["topscale"][0] if use_one_top_nuis else bkgFitResults["topscale"][ir]) )

        zjet_2b = histos[r_2b]["zjet"].Clone("zjet_2b__"+r)

        qcd_r = histos[r_2b]["data"].Clone("qcd__"+r)
        qcd_r.Add( top_2b, -1)      # added by Qi --- we still want top to be subtracted, given that their fraction is increasing in Run 2.
        qcd_r.Add( zjet_2b, -1)
        qcd_int = qcd_r.Integral()

        for ibin in range(1, qcd_r.GetNbinsX()+1):
            if qcd_r.GetBinContent(ibin) < 0:
                qcd_r.SetBinContent(ibin, 0)
                qcd_r.SetBinError(ibin, 0)


        top_r = histos[r]["top"].Clone("top__"+r)
        if (nbtag_top_shape =="3") and (r == "44") and (MassRegionName == "SR"):   # the 3b top shape is only used during the SR prediction for 44 region
            temp_scaler = top_r.Integral() / histos[r_3b]["top"].Integral()
            top_r = histos[r_3b]["top"].Clone("top__"+r)
            top_r.Scale( temp_scaler )
        top_int = top_r.Integral()

        zjet_r = histos[r]["zjet"].Clone("zjet__"+r)


        mu_qcd = bkgFitResults["muqcd"][ir]
        top_scale = (bkgFitResults["topscale"][0] if use_one_top_nuis else bkgFitResults["topscale"][ir])
        
        qcd_r.Scale( mu_qcd )
        top_r.Scale( top_scale )

        # store some numbers for table later
        e_qcd = R.Double(0.0)
        e_top = R.Double(0.0)
        Nbkg_dict[r]["qcd"] = qcd_r.IntegralAndError(0, qcd_r.GetNbinsX()+1, e_qcd)
        Nbkg_dict[r]["top"] = top_r.IntegralAndError(0, top_r.GetNbinsX()+1, e_top)
        Nbkg_dict[r]["bkg"] = Nbkg_dict[r]["qcd"] + Nbkg_dict[r]["top"]


        Nbkg_SysList[r]["qcd"].append( float(e_qcd) )
        Nbkg_SysList[r]["top"].append( float(e_top) )
        Nbkg_SysList[r]["bkg"].append( np.sqrt(float(e_qcd)**2 + float(e_top)**2) )   # Qi Question
        
        

        ## Now do smoothing ###########################################################################################
        if do_smoothing:
            ## qcd_normed = qcd_r.Clone("normed")
            ## qcd_normed.SetDirectory(0)
            ## qcd_normed.Scale(1.0 / qcd_normed.Integral())
            ## qcd_normed_sm = smoothfit.smoothfit(qcd_normed, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = True, verbose = True, outfileName="qcd_normed_smoothfit_"+r+".root")
            
            qcd_sm = smoothfit.smoothfit(qcd_r, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = True, verbose = False, outfileName="qcd_smoothfit_"+r+".root")
            top_sm = smoothfit.smoothfit(top_r, fitFunction = smoothing_func, fitRange = topSmoothRange, makePlots = True, verbose = False, outfileName="top_smoothfit_"+r+".root")
    
            if addSmoothErrorBin:
                qcd_final = smoothfit.MakeSmoothHistoWithError(qcd_r, qcd_sm)
                top_final = smoothfit.MakeSmoothHistoWithError(top_r, top_sm)
            else:
                qcd_final = smoothfit.MakeSmoothHisto(qcd_r, qcd_sm["nom"])
                top_final = smoothfit.MakeSmoothHisto(top_r, top_sm["nom"])

            qcd_final.SetNameTitle("qcd_hh_"+r+"__clone",   "qcd_hh_"+r+"__clone")
            top_final.SetNameTitle("ttbar_hh_"+r+"__clone", "ttbar_hh_"+r+"__clone")
            
        else:
            qcd_final = qcd_r.Clone("qcd_hh_"+r+"__clone")
            top_final = top_r.Clone("ttbar_hh_"+r+"__clone")

        zjet_final = zjet_r.Clone("zjet_hh_"+r+"__clone")


        if rebinFinal is not None:
            qcd_final = qcd_final.Rebin(len(rebinFinal)-1, qcd_final.GetName()+"_rebinFinal", rebinFinal)
            top_final = top_final.Rebin(len(rebinFinal)-1, top_final.GetName()+"_rebinFinal", rebinFinal)
            zjet_final = zjet_final.Rebin(len(rebinFinal)-1, zjet_final.GetName()+"_rebinFinal", rebinFinal)


        if makeOutputFiles:
            outfileStat.WriteTObject(qcd_final, "qcd_hh","Overwrite")
            outfileStat.WriteTObject(top_final, "ttbar_hh","Overwrite")
            outfileStat.WriteTObject(zjet_final, "zjet_hh","Overwrite")


        qcd_final.SetDirectory(0)
        top_final.SetDirectory(0)
        zjet_final.SetDirectory(0)
        output_Dict[r]["qcd"]["nom"] = qcd_final
        output_Dict[r]["ttbar"]["nom"] = top_final
        output_Dict[r]["zjet"]["nom"] = zjet_final

        # for systematics, don't need anything after this in loop
        if isSystematicVariation:
            continue

        ##################################################################################################################################
        ### propagate correlated systematics from the smoothing procedure---> these "replace" the stat error on the bins     #############
        ##################################################################################################################################
        if do_smoothing:
            
            ## qcd smoothing variations#################################################################
            if not addSmoothErrorBin:
                for ivar in range(len(qcd_sm["vars"])):
                    qup = qcd_sm["vars"][ivar][0]
                    qdw = qcd_sm["vars"][ivar][1]

                    qcd_r_qup = smoothfit.MakeSmoothHisto(qcd_r, qup)
                    qcd_r_qdw = smoothfit.MakeSmoothHisto(qcd_r, qdw)

                    qcd_r_qup.SetNameTitle("qcd_hh_"+r+"_smoothQ"+str(ivar)+"Up__clone", "qcd_hh_"+r+"_smoothQ"+str(ivar)+"Up__clone")
                    qcd_r_qdw.SetNameTitle("qcd_hh_"+r+"_smoothQ"+str(ivar)+"Down__clone", "qcd_hh_"+r+"_smoothQ"+str(ivar)+"Down__clone")


                    if rebinFinal is not None:
                        qcd_r_qup = qcd_r_qup.Rebin(len(rebinFinal)-1, qcd_r_qup.GetName()+"_rebinFinal", rebinFinal)
                        qcd_r_qdw = qcd_r_qdw.Rebin(len(rebinFinal)-1, qcd_r_qdw.GetName()+"_rebinFinal", rebinFinal)

                    if makeOutputFiles:
                        outfileStat.WriteTObject(qcd_r_qup, "qcd_hh_smoothQ"+str(ivar)+"Up","Overwrite")
                        outfileStat.WriteTObject(qcd_r_qdw, "qcd_hh_smoothQ"+str(ivar)+"Down","Overwrite")

                    qcd_r_qup.SetDirectory(0)
                    qcd_r_qdw.SetDirectory(0)
                    output_Dict[r]["qcd"]["smoothQ"+str(ivar)+"Up"] = qcd_r_qup
                    output_Dict[r]["qcd"]["smoothQ"+str(ivar)+"Down"] = qcd_r_qdw


                
            ## qcd smoothing function variations #################################################################
            if smoothing_func == "ExpModGauss":
                smoothFuncCompSyst = EMGSmoothSyst.smoothFuncCompare(qcd_r, fitFunction = smoothing_func,
                                                                     fitRange = qcdSmoothRange, funcCompareRange=(900, qcdSmoothRange[1]),
                                                                     makePlots = True, verbose = False, outfileName="EMGSmoothFuncCompare_"+r+".root", plotExtra=True)  # Qi
            else:
                # smoothFuncCompSyst = smoothfit.smoothFuncCompare(qcd_r, fitRange = (900, qcdSmoothRange[1]),
                smoothFuncCompSyst = smoothfit.smoothFuncCompare(qcd_r, fitRange = (qcdSmoothRange[0], qcdSmoothRange[1]),            # qi
                                                                 makePlots = True, verbose = False, outfileName="smoothFuncCompare_"+r+".root", plotExtra=False)  # Qi
                
            qcd_r_func_up = smoothFuncCompSyst["up"]
            qcd_r_func_dw = smoothFuncCompSyst["dw"]
            qcd_r_func_up_super = smoothFuncCompSyst["up_super"]
            qcd_r_func_dw_super = smoothFuncCompSyst["dw_super"]

            if rebinFinal is not None:
                qcd_r_func_up = qcd_r_func_up.Rebin(len(rebinFinal)-1, qcd_r_func_up.GetName()+"_rebinFinal", rebinFinal)
                qcd_r_func_dw = qcd_r_func_dw.Rebin(len(rebinFinal)-1, qcd_r_func_dw.GetName()+"_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(qcd_r_func_up, "qcd_hh_smoothFuncUp","Overwrite")
                outfileStat.WriteTObject(qcd_r_func_dw, "qcd_hh_smoothFuncDown","Overwrite")
                
                outfileStat.WriteTObject(qcd_r_func_up_super, "qcd_hh_smoothFuncSuperUp","Overwrite")
                outfileStat.WriteTObject(qcd_r_func_dw_super, "qcd_hh_smoothFuncSuperDown","Overwrite")

            qcd_r_func_up.SetDirectory(0)
            qcd_r_func_dw.SetDirectory(0)
            output_Dict[r]["qcd"]["smoothFuncUp"] = qcd_r_func_up
            output_Dict[r]["qcd"]["smoothFuncDown"] = qcd_r_func_dw

            qcd_r_func_up_super.SetDirectory(0)
            qcd_r_func_dw_super.SetDirectory(0)
            output_Dict[r]["qcd"]["smoothFuncUp_super"] = qcd_r_func_up_super
            output_Dict[r]["qcd"]["smoothFuncDown_super"] = qcd_r_func_dw_super
            
            #smoothfit.smoothFuncRangeCompare(qcd_r, fitRange = (900, qcdSmoothRange[1]), makePlots = True, verbose = False, outfileName="smoothFuncRangeCompare_"+r+".root")
            
            smoothfit.smoothFuncRangeCompare(qcd_r, fitFunction = smoothing_func, fitRange = qcdSmoothRange, fitMaxVals = ["3000", "2500", "2000", "1700"],
                                            makePlots = True, plotExtra = False, verbose = False, outfileName="smoothFuncRangeCompare_"+r+".root")   # Qi
            
            ## ttbar smoothing variations##############################################################################
            if not addSmoothErrorBin:
                for ivar in range(len(top_sm["vars"])):
                    tup = top_sm["vars"][ivar][0]
                    tdw = top_sm["vars"][ivar][1]

                    top_r_tup = smoothfit.MakeSmoothHisto(top_r, tup)
                    top_r_tdw = smoothfit.MakeSmoothHisto(top_r, tdw)

                    top_r_tup.SetNameTitle("ttbar_hh_"+r+"_smoothT"+str(ivar)+"Up__clone",   "ttbar_hh_"+r+"_smoothT"+str(ivar)+"Up__clone")
                    top_r_tdw.SetNameTitle("ttbar_hh_"+r+"_smoothT"+str(ivar)+"Down__clone", "ttbar_hh_"+r+"_smoothT"+str(ivar)+"Down__clone")

                    if rebinFinal is not None:
                        top_r_tup = top_r_tup.Rebin(len(rebinFinal)-1, top_r_tup.GetName()+"_rebinFinal", rebinFinal)
                        top_r_tdw = top_r_tdw.Rebin(len(rebinFinal)-1, top_r_tdw.GetName()+"_rebinFinal", rebinFinal)

                    if makeOutputFiles:
                        outfileStat.WriteTObject(top_r_tup, "ttbar_hh_smoothT"+str(ivar)+"Up","Overwrite")
                        outfileStat.WriteTObject(top_r_tdw, "ttbar_hh_smoothT"+str(ivar)+"Down","Overwrite")

                    top_r_tup.SetDirectory(0)
                    top_r_tdw.SetDirectory(0)
                    output_Dict[r]["ttbar"]["smoothT"+str(ivar)+"Up"] = top_r_tup
                    output_Dict[r]["ttbar"]["smoothT"+str(ivar)+"Down"] = top_r_tdw

            

        ########################################################################################################
        ### propagate correlated systematics from normalization fits for mu_qcd and top_scale    ###############
        ########################################################################################################
        for ivar in range(len(pvars)):
            sys_qcd = []
            sys_top = []
            sys_bkg = []
            for iUD in range(2):
                UpDw = ("Up" if iUD ==0 else "Down")
                
                mu_qcd_var = pvars[ivar][iUD][ir]
                top_scale_var = pvars[ivar][iUD][n_channels + (0 if use_one_top_nuis else ir) ]

                qvar = qcd_r.Clone("qvar")
                qvar.Scale( mu_qcd_var * qcd_int / qvar.Integral() )

                ## for ibin in range(1, qvar.GetNbinsX()+1):
                ##     if qvar.GetBinError(ibin) > qvar.GetBinContent(ibin):
                ##         qvar.SetBinError(ibin, qvar.GetBinContent(ibin))
                
                tvar = top_r.Clone("tvar")
                tvar.Scale( top_scale_var * top_int / tvar.Integral() )

                ### store some numbers for table
                sys_qcd.append( qvar.Integral() - Nbkg_dict[r]["qcd"] )
                sys_top.append( tvar.Integral() - Nbkg_dict[r]["top"] )
                sys_bkg.append( qvar.Integral() + tvar.Integral() - Nbkg_dict[r]["bkg"] )

                #vartxt = vartxt + str(r) + ' ' + str(ivar) + ' ' + str(iUD) + ' ' + str(qvar.Integral()) + ' ' + str(tvar.Integral()) + ' ' + str( (qvar.Integral() + tvar.Integral())) + '\n'

                ## Now do smoothing #######
                if do_smoothing:
                    qvar_sm = smoothfit.smoothfit(qvar, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = False, verbose = verbose,
                                                  outfileName="qcd_smoothfit_"+r+"_Norm"+str(ivar)+str(iUD)+".root")
                    tvar_sm = smoothfit.smoothfit(tvar, fitFunction = smoothing_func, fitRange = topSmoothRange, makePlots = False, verbose = verbose,
                                                  outfileName="top_smoothfit_"+r+"_Norm"+str(ivar)+str(iUD)+".root")

                    if addSmoothErrorBin:
                        qvar_final = smoothfit.MakeSmoothHistoWithError(qvar, qvar_sm)
                        tvar_final = smoothfit.MakeSmoothHistoWithError(tvar, tvar_sm)
                    else:
                        qvar_final = smoothfit.MakeSmoothHisto(qvar, qvar_sm["nom"])
                        tvar_final = smoothfit.MakeSmoothHisto(tvar, tvar_sm["nom"])

                    qvar_final.SetNameTitle("qcd_hh_"+r+"_normY"+str(ivar)+UpDw+"__clone",   "qcd_hh_"+r+"_normY"+str(ivar)+UpDw+"__clone")
                    tvar_final.SetNameTitle("ttbar_hh_"+r+"_normY"+str(ivar)+UpDw+"__clone", "ttbar_hh_"+r+"_normY"+str(ivar)+UpDw+"__clone")

                else:
                    qvar_final = qvar.Clone("qcd_hh_"+r+"_normY"+str(ivar)+UpDw+"__clone")
                    tvar_final = tvar.Clone("ttbar_hh_"+r+"_normY"+str(ivar)+UpDw+"__clone")

                if rebinFinal is not None:
                    qvar_final = qvar_final.Rebin(len(rebinFinal)-1, qvar_final.GetName()+"_rebinFinal", rebinFinal)
                    tvar_final = tvar_final.Rebin(len(rebinFinal)-1, tvar_final.GetName()+"_rebinFinal", rebinFinal)

                if makeOutputFiles:
                    outfileStat.WriteTObject(qvar_final, "qcd_hh_normY"+str(ivar)+UpDw,"Overwrite")
                    outfileStat.WriteTObject(tvar_final, "ttbar_hh_normY"+str(ivar)+UpDw,"Overwrite")

                qvar_final.SetDirectory(0)
                tvar_final.SetDirectory(0)
                output_Dict[r]["qcd"]["normY"+str(ivar)+UpDw] = qvar_final
                output_Dict[r]["ttbar"]["normY"+str(ivar)+UpDw] = tvar_final

                
            # store some numbers for table later
            e_qcd_i = np.max( np.abs(sys_qcd) )
            e_top_i = np.max( np.abs(sys_top) )
            e_bkg_i = np.max( np.abs(sys_bkg) )


            Nbkg_SysList[r]["qcd"].append( e_qcd_i )
            Nbkg_SysList[r]["top"].append( e_top_i )
            Nbkg_SysList[r]["bkg"].append( e_bkg_i )



            
        ########################################################################################################
        ####### QCD Shape and Norm estimated from CR            ################################################
        ########################################################################################################
        if QCDSyst_Dict!=None and (distributionName=="DiJetMass" or distributionName=="DiJetMassPrime"):  # qi
            
            qvar_shape_up = qcd_r.Clone("qvar_QCDshape_up")
            qvar_shape_up.Multiply( QCDSyst_Dict["Shape_"+r]["fup"] )

            qvar_shape_dw = qcd_r.Clone("qvar_QCDshape_dw")
            qvar_shape_dw.Multiply( QCDSyst_Dict["Shape_"+r]["fdw"] )

            for ibinX in range(1, qvar_shape_up.GetNbinsX()+1):
                if(qvar_shape_up.GetBinContent(ibinX) < 0):
                    qvar_shape_up.SetBinContent(ibinX, 0)
                    qvar_shape_up.SetBinError(ibinX, 0)

                if(qvar_shape_dw.GetBinContent(ibinX) < 0):
                    qvar_shape_dw.SetBinContent(ibinX, 0)
                    qvar_shape_dw.SetBinError(ibinX, 0)
                    

            qvar_shape_up.Scale( qcd_r.Integral() / qvar_shape_up.Integral() )
            qvar_shape_dw.Scale( qcd_r.Integral() / qvar_shape_dw.Integral() )
        


            ## Now do smoothing
            if do_smoothing:
                qvar_shape_up_sm = smoothfit.smoothfit(qvar_shape_up, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = False, verbose = verbose,
                                                        outfileName="qcd_smoothfit_"+r+"_QCDShapeUp.root")

                qvar_shape_dw_sm = smoothfit.smoothfit(qvar_shape_dw, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = False, verbose = verbose,
                                                        outfileName="qcd_smoothfit_"+r+"_QCDShapeDown.root")

                if addSmoothErrorBin:
                    qvar_shape_up_final = smoothfit.MakeSmoothHistoWithError(qvar_shape_up, qvar_shape_up_sm)
                    qvar_shape_dw_final = smoothfit.MakeSmoothHistoWithError(qvar_shape_dw, qvar_shape_dw_sm)
                else:
                    qvar_shape_up_final = smoothfit.MakeSmoothHisto(qvar_shape_up, qvar_shape_up_sm["nom"])
                    qvar_shape_dw_final = smoothfit.MakeSmoothHisto(qvar_shape_dw, qvar_shape_dw_sm["nom"])

                qvar_shape_up_final.SetNameTitle("qcd_hh_"+r+"_QCDShapeCRUp__clone",     "qcd_hh_"+r+"_QCDShapeCRUp__clone")
                qvar_shape_dw_final.SetNameTitle("qcd_hh_"+r+"_QCDShapeCRDown__clone",   "qcd_hh_"+r+"_QCDShapeCRDown__clone")


            else:
                qvar_shape_up_final = qvar_shape_up.Clone("qcd_hh_"+r+"_QCDShapeCRUp__clone")
                qvar_shape_dw_final = qvar_shape_dw.Clone("qcd_hh_"+r+"_QCDShapeCRDown__clone")


            if rebinFinal is not None:
                qvar_shape_up_final = qvar_shape_up_final.Rebin(len(rebinFinal)-1, qvar_shape_up_final.GetName()+"_rebinFinal", rebinFinal)
                qvar_shape_dw_final = qvar_shape_dw_final.Rebin(len(rebinFinal)-1, qvar_shape_dw_final.GetName()+"_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(qvar_shape_up_final, "qcd_hh_QCDShapeCRUp")
                outfileStat.WriteTObject(qvar_shape_dw_final, "qcd_hh_QCDShapeCRDown")

            qvar_shape_up_final.SetDirectory(0)
            qvar_shape_dw_final.SetDirectory(0)
            output_Dict[r]["qcd"]["QCDShapeCRUp"] = qvar_shape_up_final
            output_Dict[r]["qcd"]["QCDShapeCRDown"] = qvar_shape_dw_final


        ###########################################################################################
        ### Norm comparison in CR      ############################################################
        ###########################################################################################
        if QCDSyst_Dict != None:
            
            qvar_normCR_up =  qcd_final.Clone("qcd_hh_"+r+"_QCDnormCRUp__clone")
            qvar_normCR_up.Scale( 1.0 + QCDSyst_Dict["Scale_"+r] )

            qvar_normCR_dw =  qcd_final.Clone("qcd_hh_"+r+"_QCDnormCRDown__clone")
            qvar_normCR_dw.Scale( 1.0 - QCDSyst_Dict["Scale_"+r] )

            if rebinFinal is not None:
                qvar_normCR_up = qvar_normCR_up.Rebin(len(rebinFinal)-1, qvar_normCR_up.GetName()+"_rebinFinal", rebinFinal)
                qvar_normCR_dw = qvar_normCR_dw.Rebin(len(rebinFinal)-1, qvar_normCR_dw.GetName()+"_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(qvar_normCR_up, "qcd_hh_QCDNormCRUp")
                outfileStat.WriteTObject(qvar_normCR_dw, "qcd_hh_QCDNormCRDown")

            qvar_normCR_up.SetDirectory(0)
            qvar_normCR_dw.SetDirectory(0)
            output_Dict[r]["qcd"]["QCDNormCRUp"] = qvar_normCR_up
            output_Dict[r]["qcd"]["QCDNormCRDown"] = qvar_normCR_dw


            
        #####################################################################################################################
        ### top shape systematics in 4b region, if using 3b shape ###########################################################
        #####################################################################################################################
        if r == "44" and nbtag_top_shape_SRPred == "3" and MassRegionName == "SR"  and (distributionName=="DiJetMass" or distributionName=="DiJetMassPrime"):   # qi
            ttbarShapeSRSyst_Dict = SystTools.ttbarShapeSysSR(topfileName,
                                                                distributionName,
                                                                signal_region = "43",
                                                                compare_region = "44",
                                                                btag_WP     = btag_WP,
                                                                makePlots = True,
                                                                verbose = False,
                                                                outfileNameBase="TopShapeSRSysfit.root")

            tvar_shape_up = top_r.Clone("tvar_ttbarShapeSR_up")
            tvar_shape_up.Multiply( ttbarShapeSRSyst_Dict["fup"] )

            tvar_shape_dw = top_r.Clone("tvar_ttbarShapeSR_dw")
            tvar_shape_dw.Multiply( ttbarShapeSRSyst_Dict["fdw"] )


            for ibinX in range(1, tvar_shape_up.GetNbinsX()+1):
                if(tvar_shape_up.GetBinContent(ibinX) < 0):
                    tvar_shape_up.SetBinContent(ibinX, 0)
                    tvar_shape_up.SetBinError(ibinX, 0)

                if(tvar_shape_dw.GetBinContent(ibinX) < 0):
                    tvar_shape_dw.SetBinContent(ibinX, 0)
                    tvar_shape_dw.SetBinError(ibinX, 0)
                    

            tvar_shape_up.Scale( top_r.Integral() / tvar_shape_up.Integral() )
            tvar_shape_dw.Scale( top_r.Integral() / tvar_shape_dw.Integral() )


            ## Now do smoothing ##########################
            if do_smoothing:
                tvar_shape_up_sm = smoothfit.smoothfit(tvar_shape_up, fitFunction = smoothing_func, fitRange = topSmoothRange, makePlots = False, verbose = verbose,
                                                        outfileName="top_smoothfit_"+r+"_ttbarShapeSRUp.root")

                tvar_shape_dw_sm = smoothfit.smoothfit(tvar_shape_dw, fitFunction = smoothing_func, fitRange = topSmoothRange, makePlots = False, verbose = verbose,
                                                        outfileName="top_smoothfit_"+r+"_ttbarShapeSReDown.root")

                if addSmoothErrorBin:
                    tvar_shape_up_final = smoothfit.MakeSmoothHistoWithError(tvar_shape_up, tvar_shape_up_sm)
                    tvar_shape_dw_final = smoothfit.MakeSmoothHistoWithError(tvar_shape_dw, tvar_shape_dw_sm)
                else:
                    tvar_shape_up_final = smoothfit.MakeSmoothHisto(tvar_shape_up, tvar_shape_up_sm["nom"])
                    tvar_shape_dw_final = smoothfit.MakeSmoothHisto(tvar_shape_dw, tvar_shape_dw_sm["nom"])

                tvar_shape_up_final.SetNameTitle("ttbar_hh_"+r+"_ttbarShapeSRUp__clone",     "ttbar_hh_"+r+"_ttbarShapeSRUp__clone")
                tvar_shape_dw_final.SetNameTitle("ttbar_hh_"+r+"_ttbarShapeSRDown__clone",   "ttbar_hh_"+r+"_ttbarShapeSRDown__clone")


            else:
                tvar_shape_up_final = tvar_shape_up.Clone("ttbar_hh_"+r+"_ttbarShapeSRUp__clone")
                tvar_shape_dw_final = tvar_shape_dw.Clone("ttbar_hh_"+r+"_ttbarShapeSRDown__clone")


            if rebinFinal is not None:
                tvar_shape_up_final = tvar_shape_up_final.Rebin(len(rebinFinal)-1, tvar_shape_up_final.GetName()+"_rebinFinal", rebinFinal)
                tvar_shape_dw_final = tvar_shape_dw_final.Rebin(len(rebinFinal)-1, tvar_shape_dw_final.GetName()+"_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(tvar_shape_up_final, "ttbar_hh_ttbarShapeSRUp")
                outfileStat.WriteTObject(tvar_shape_dw_final, "ttbar_hh_ttbarShapeSRDown")

            tvar_shape_up_final.SetDirectory(0)
            tvar_shape_dw_final.SetDirectory(0)
            output_Dict[r]["ttbar"]["ttbarShapeSRUp"] = tvar_shape_up_final
            output_Dict[r]["ttbar"]["ttbarShapeSRDown"] = tvar_shape_dw_final

            


        
        ### close outfiles, if used ###
        if makeOutputFiles:
            outfileStat.Close()

    ### Print tables ###
    #PrintTable( Nbkg_dict, Nbkg_SysList, regions)
    #print vartxt

    #print output_Dict

    output_Dict['regions'] = regions

    return output_Dict
def QCDSystematics(datafileName="hist_data.root",
                    topfileName="hist_ttbar.root",
                    zjetfileName="hist_Zjets.root",
                    distributionName= "mHH_l",
                    n_trkjet  = ["4","3","2"],
                    n_btag    = ["4","3","2"],
                    btag_WP     = "77",
                    mu_qcd_vals = [1.0, 1.0],
                    topscale_vals = [1.0, 1.0],
                    NRebin = 1,
                    smoothing_func = "Dijet",
                    SmoothRange = (1100, 3000),# (100, 2500),
                    use_one_top_nuis = False,
                    use_scale_top_0b = False,
                    nbtag_top_shape_for4b = None,
                    makePlots = False,
                    verbose = False,
                    outfileNameBase="QCDSysfitSmooth.root"):
    ##this cannot have the norm fixed
    # global rfunc1
    # global rfunc2
    ##### Parse Inputs ############################################
    dist_name   = distributionName
    
    num_trkjet  = np.asarray(n_trkjet)
    if num_trkjet.shape==():
        num_trkjet = np.asarray([n_trkjet])

    num_btag    = np.asarray(n_btag)
    if num_btag.shape==():
        num_btag = np.asarray([n_btag])
    if num_btag.shape!=num_trkjet.shape:
        print "Must have same number of track jet and b-tag regions specified"
        sys.exit(0)
  
    btag_WP     = btag_WP
    
    n_rebin     = NRebin

    useOneTopNuis = use_one_top_nuis

    scaleTop0b = use_scale_top_0b

    n_channels = num_trkjet.shape[0]

    regions = [ num_trkjet[i]+num_btag[i] for i in range(n_channels) ]
    ##################################################################


    colorlist = [ R.kGreen, R.kOrange, R.kMagenta, R.kCyan, R.kPink, (R.kAzure+1), R.kGreen+2, R.kOrange+5]        


    ##### Get Signal Region Histograms ################################
    datafile = R.TFile(datafileName,"READ")
    topfile  = R.TFile(topfileName,"READ")
    zjetfile  = ( R.TFile(zjetfileName,"READ") if zjetfileName!=None else None)


    histos = {}
    
    # collect all histograms
    for r in ["44","33","22","40","30","20"]:
        folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP, "CR")  #folder( r[0], r[1], btag_WP)
        #print folder_r
        data_r   = datafile.Get(folder_r).Clone("data_"+r)
        data_r.SetDirectory(0)
        
        top_r    = topfile.Get(folder_r).Clone("top_"+r)
        top_r.SetDirectory(0)

        zjet_r   = CheckAndGet(zjetfile, folder_r, top_r).Clone("zjet_"+r)
        zjet_r.SetDirectory(0)

        for ibin in range(1, top_r.GetNbinsX()+1):
            if top_r.GetBinContent(ibin) < 0:
                top_r.SetBinContent(ibin, 0)
                top_r.SetBinError(ibin, 0)

        data_r.Rebin(n_rebin)
        top_r.Rebin(n_rebin)
        zjet_r.Rebin(n_rebin)


        histos[r]     = {"data": data_r,  "top": top_r,  "zjet":zjet_r}

    datafile.Close()
    topfile.Close()
    if zjetfile != None:
        zjetfile.Close()
    ##################################################################

    ####### outpue object ###################
    QCDSyst_Dict = {}
    

    ##### scaling and subtractions #################################
    for ir in range(len(regions)):
        r = regions[ir]
                
        r_0b = r[0]+"0"

        top_0b = histos[r_0b]["top"].Clone("top_0b__"+r)
        if scaleTop0b:
            top_0b.Scale( (topscale_vals[0] if use_one_top_nuis else topscale_vals[ir]) )

        zjet_0b = histos[r_0b]["zjet"].Clone("zjet_0b__"+r)


        qcd_r = histos[r_0b]["data"].Clone("qcd__"+r)
        qcd_r.Add( top_0b, -1)      # added by Qi --- we still want top to be subtracted, given that their fraction is increasing in Run 2.
        qcd_r.Add( zjet_0b, -1)
        qcd_int = qcd_r.Integral()

        if nbtag_top_shape_for4b != None and (r == "44" or r == "33"): ##this is fine; replace everything with 2bs now
            top_r = histos[nbtag_top_shape_for4b]["top"].Clone("top__"+r)
            top_r.Scale( histos[r]["top"].Integral() / top_r.Integral() ) #scale to correct norm for region
        else:
            top_r = histos[r]["top"].Clone("top__"+r)

        top_int = top_r.Integral()

        zjet_r = histos[r]["zjet"].Clone("zjet__"+r)



        mu_qcd = mu_qcd_vals[ir]
        top_scale = (topscale_vals[0] if use_one_top_nuis else topscale_vals[ir])
        
        qcd_r.Scale( mu_qcd )
        top_r.Scale( top_scale )

        N_qcd_r = qcd_r.Integral()

        #now do ratio
        bkg_r = qcd_r.Clone("bkg__"+r)
        bkg_r.Add( top_r )
        #bkg_r.Add( zjet_r )
        bkg_r.Add( zjet_r )
        ClearNegBin(bkg_r)
        
        # for j in range(bkg_r.GetNbinsX()):
        #     print bkg_r.GetName(), bkg_r.GetBinContent(j), bkg_r.GetBinCenter(j)


        N_bkg_r         = bkg_r.Integral()

        Err_N_data_CR_r = R.Double(0)
        N_data_CR_r     =  histos[r]["data"].IntegralAndError(0, histos[r]["data"].GetNbinsX()+1, Err_N_data_CR_r)

        ##rescale the background estimation to have the same norm?
        bkg_r.Scale(histos[r]["data"].Integral() / bkg_r.Integral())


        c=R.TCanvas("c1_cr_"+r,"c1_cr_"+r)
        xleg, yleg = 0.52, 0.7
        leg = R.TLegend(xleg, yleg, xleg+0.3, yleg+0.2)
        leg.SetFillColor(0)
        leg.SetBorderSize(0)
        leg.SetMargin(0.3)
        histos[r]["data"].SetXTitle("m_{JJ} [GeV]")
        histos[r]["data"].SetYTitle("Entries")
        histos[r]["data"].GetXaxis().SetRangeUser(500, 4000)
        histos[r]["data"].Draw("E1")
        leg.AddEntry(histos[r]["data"], "CR data", "LP")

        ##################################
        ## smooth bkg and data
        ##################################
        data_sm   = smoothfit.smoothfit(histos[r]["data"], fitFunction = smoothing_func, fitRange = SmoothRange, makePlots = False, verbose = False, useLikelihood=True, outfileName="data_smoothfit_CRsyst_"+r+".root")
        data_sm_h = smoothfit.MakeSmoothHisto(histos[r]["data"], data_sm["nom"], keepNorm=False)

        data_sm["nom"].SetNameTitle("data_smoothfit_CRsyst_"+r,"data_smoothfit_CRsyst_"+r)
        data_sm["nom"].SetLineColor(R.kBlack)
        data_sm["nom"].Draw("same")
        leg.AddEntry(data_sm["nom"], "CR data smoothed", "L")

        bkg_sm   = smoothfit.smoothfit(bkg_r, fitFunction = smoothing_func, fitRange = SmoothRange, makePlots = False, verbose = False, outfileName="bkg_smoothfit_CRsyst_"+r+".root")
        bkg_sm_h = smoothfit.MakeSmoothHisto(bkg_r, bkg_sm["nom"], keepNorm=False)

        bkg_sm["nom"].SetNameTitle("bkg_smoothfit_CRsyst_"+r,"bkg_smoothfit_CRsyst_"+r)
        bkg_sm["nom"].SetLineColor(R.kBlue)
        bkg_sm["nom"].Draw("same")
        leg.AddEntry(bkg_sm["nom"], "CR Prediction smoothed", "L")


  
        rfunc1 = data_sm["nom"]
        rfunc2 = bkg_sm["nom"]
        def rfunc_ratio(x):
            return rfunc1.Eval(x[0]) / rfunc2.Eval(x[0])
        xMax   = histos[r]["data"].GetXaxis().GetBinUpEdge(histos[r]["data"].GetXaxis().GetNbins())
        ratio_sm = R.TF1("ratio_crsys_sm"+r, rfunc_ratio, SmoothRange[0], xMax, 0)
        ## ratio_sm.SetLineColor(R.kGray)

        for ivar in range(len(bkg_sm["vars"])):
            dup = bkg_sm["vars"][ivar][0]
            ddw = bkg_sm["vars"][ivar][1]

            bkg_r_qup = smoothfit.MakeSmoothHisto(bkg_r, dup, keepNorm=False)
            bkg_r_qdw = smoothfit.MakeSmoothHisto(bkg_r, ddw, keepNorm=False)

            for ibin in range(1,  bkg_sm_h.GetNbinsX()+1):
                err_val = np.max( np.abs( [ bkg_sm_h.GetBinContent(ibin) - bkg_r_qup.GetBinContent(ibin), bkg_sm_h.GetBinContent(ibin) - bkg_r_qdw.GetBinContent(ibin)] ) )
                bkg_sm_h.SetBinError(ibin, np.sqrt( bkg_sm_h.GetBinError(ibin)**2 + err_val**2) )


        for ivar in range(len(data_sm["vars"])):
            dup = data_sm["vars"][ivar][0]
            ddw = data_sm["vars"][ivar][1]

            dup.SetLineColor(colorlist[ivar])
            ddw.SetLineColor(colorlist[ivar])

            dup.Draw("same")
            ddw.Draw("same")
            leg.AddEntry(dup, "CR data smoothed variation", "L")


            data_r_qup = smoothfit.MakeSmoothHisto(histos[r]["data"], dup, keepNorm=False)
            data_r_qdw = smoothfit.MakeSmoothHisto(histos[r]["data"], ddw, keepNorm=False)

            for ibin in range(1,  data_sm_h.GetNbinsX()+1):
                err_val = np.max( np.abs( [ data_sm_h.GetBinContent(ibin) - data_r_qup.GetBinContent(ibin), data_sm_h.GetBinContent(ibin) - data_r_qdw.GetBinContent(ibin)] ) )
                data_sm_h.SetBinError(ibin, np.sqrt( data_sm_h.GetBinError(ibin)**2 + err_val**2) )


        c.SetLogy(1)
        leg.Draw("same")
        c.SaveAs(outfileNameBase.split(".root")[0] + "_" + r + ".root")
        c.SaveAs(outfileNameBase.split(".root")[0] + "_" + r + ".pdf")
        c.Close()


        h_ratio_cr_nom = data_sm_h.Clone("data_sm_h_CRsyst_"+r)
        h_ratio_cr_nom.Divide( data_sm["nom"] )
        h_ratio_cr_nom.SetDirectory(0)
                
        h_ratio_cr = data_sm_h.Clone("data_sm_h_CRsyst2_"+r)
        h_ratio_cr.Divide( bkg_sm_h )
        h_ratio_cr.SetDirectory(0)


        QCDSyst_Dict["Shape_"+r] = ratio_sm.Clone(ratio_sm.GetName() + r) ##this was a huge bug...

        #scale is max of ratio non-unity and CR stat error 
        QCDSyst_Dict["Scale_"+r] = np.max( np.abs( [ (N_bkg_r - N_data_CR_r)/N_bkg_r,  (Err_N_data_CR_r / N_data_CR_r), _extraNormCRSysDict.get(r, 0.) ] ) )
        #print "Scale_"+r, QCDSyst_Dict["Scale_"+r], N_bkg_r, N_data_CR_r, Err_N_data_CR_r,  (N_bkg_r - N_data_CR_r)/N_bkg_r, Err_N_data_CR_r / N_data_CR_r
        
        
        c2=R.TCanvas("c2_cr_"+r,"c2_cr_"+r)
        leg = R.TLegend(0.2,0.7,0.5,0.9)
        leg.SetFillColor(0)
        h_ratio_cr_nom.SetFillColor(R.kBlue)
        h_ratio_cr_nom.SetFillStyle(3004)
        h_ratio_cr_nom.SetMarkerSize(0)
        h_ratio_cr_nom.GetXaxis().SetRangeUser(1200, 4000)
        h_ratio_cr_nom.GetYaxis().SetRangeUser(0, 5)
        h_ratio_cr_nom.GetXaxis().SetLabelSize(0.04)
        h_ratio_cr_nom.GetYaxis().SetLabelSize(0.04)
        h_ratio_cr_nom.SetXTitle("m_{JJ} [GeV]")
        h_ratio_cr_nom.SetYTitle("Ratio")
        h_ratio_cr_nom.Draw("E2")
        leg.AddEntry(h_ratio_cr_nom, "CR data", "F")


        h_ratio_cr.SetLineColor(R.kBlack)
        h_ratio_cr.SetFillColor(R.kBlack)
        h_ratio_cr.SetMarkerSize(0)
        h_ratio_cr.SetFillStyle(3003)
        h_ratio_cr.Draw("same E2")
        leg.AddEntry(h_ratio_cr, "CR Prediction", "LF")


        ratio_sm.Draw("same")
        leg.Draw("same")
        c2.SaveAs(outfileNameBase.split(".root")[0] + "_ratio_" + r + ".root")
        c2.SaveAs(outfileNameBase.split(".root")[0] + "_ratio_" + r + ".pdf")
        c2.Close()

        ratio_sm = None
        rfunc1 = None
        rfunc2 = None

    datafile.Close()
    topfile.Close()
    
    return QCDSyst_Dict
예제 #5
0
def HistoAnalysis(
        datafileName="data/hist_data.root",
        topfileName="data/hist_ttbar.root",
        zjetfileName=None,
        distributionName="mHH_pole",
        n_trkjet=["4", "3", "2"],
        n_btag=["4", "3", "2"],
        btag_WP="77",
        NRebin=10,
        use_one_top_nuis=False,
        use_scale_top_0b=False,
        nbtag_top_shape_normFit_for4b="33",
        nbtag_top_shape_SRPred_for4b="33",
        rebinFinal=None,
        smoothing_func="Dijet",
        top_smoothing_func="Dijet",
        inputFitResult=None,
        inputQCDSyst_Dict=None,
        doSmoothing=True,
        addSmoothErrorBin=False,
        qcdSmoothRange=(1200, 3000),  # (100, 2500),
        topSmoothRange=(1200, 3000),  #(100, 2000),
        isSystematicVariation=False,
        verbose=False,
        makeOutputFiles=True,
        MassRegionName="SR"):

    global func1
    global func2

    ##### Parse Inputs ############################################
    dist_name = distributionName

    num_trkjet = np.asarray(n_trkjet)
    if num_trkjet.shape == ():
        num_trkjet = np.asarray([n_trkjet])

    num_btag = np.asarray(n_btag)
    if num_btag.shape == ():
        num_btag = np.asarray([n_btag])
    if num_btag.shape != num_trkjet.shape:
        print "Must have same number of track jet and b-tag regions specified"
        sys.exit(0)

    btag_WP = btag_WP

    n_rebin = NRebin

    nbtag_top_shape_for4b = nbtag_top_shape_SRPred_for4b
    topShape_nbtag_for4b = nbtag_top_shape_for4b
    if nbtag_top_shape_for4b == None:
        topShape_nbtag_for4b = num_btag + num_btag

    useOneTopNuis = use_one_top_nuis

    scaleTop0b = use_scale_top_0b

    n_channels = num_trkjet.shape[0]

    regions = [num_trkjet[i] + num_btag[i] for i in range(n_channels)]

    ##for outputing
    isMhhDistribution = (distributionName == "mHH_l"
                         or distributionName == "mHH_pole")

    do_smoothing = (doSmoothing if isMhhDistribution else False)  # qi

    ##################################################################

    ##### Storage Variables ############################################
    output_Dict = {}

    Nbkg_dict = {}
    Nbkg_SysList = {}
    for ir in regions:
        Nbkg_dict[ir] = {"qcd": 0, "top": 0, "zjet": 0, "bkg": 0}
        Nbkg_SysList[ir] = {"qcd": [], "top": [], "zjet": [], "bkg": []}

    vartxt = ''
    ##################################################################

    ##### Do Background Fits ############################################
    if inputFitResult == None:
        bkgFitResults = BkgFit.BackgroundFit(
            datafileName=datafileName,
            topfileName=topfileName,
            zjetfileName=zjetfileName,
            distributionName="leadHCand_Mass",
            n_trkjet=n_trkjet,
            n_btag=n_btag,
            btag_WP=btag_WP,
            NRebin=2,  #NRebin,
            use_one_top_nuis=use_one_top_nuis,
            use_scale_top_0b=use_scale_top_0b,
            nbtag_top_shape_for4b=nbtag_top_shape_normFit_for4b,
            makePlots=True,
            verbose=verbose)

    else:
        bkgFitResults = inputFitResult

    pvars = bkgFitResults["pvars"]

    output_Dict["fitResults"] = bkgFitResults
    ##################################################################

    ##### Get QCD Shape Systematics from CR  ##############################
    if MassRegionName == "SR":
        # should only affect SR
        if inputQCDSyst_Dict == None and isMhhDistribution:  # qi
            QCDSyst_Dict = SystToolsSmooth.QCDSystematics(
                datafileName=datafileName,
                topfileName=topfileName,
                zjetfileName=zjetfileName,
                distributionName=
                "mHH_l",  # this has been decided to fix on DiJetMass
                n_trkjet=n_trkjet,
                n_btag=n_btag,
                btag_WP=btag_WP,
                mu_qcd_vals=bkgFitResults["muqcd"],
                topscale_vals=bkgFitResults["topscale"],
                NRebin=5,
                smoothing_func=smoothing_func,
                SmoothRange=(1100, 3000),  # (100, 2500),
                use_one_top_nuis=use_one_top_nuis,
                use_scale_top_0b=use_scale_top_0b,
                nbtag_top_shape_for4b=nbtag_top_shape_SRPred_for4b,
                makePlots=True,
                verbose=False,
                outfileNameBase="QCDSysfitSmooth.root")

            ## QCDSyst_Dict = SystTools.QCDSystematics(datafileName=datafileName,
            ##                                         topfileName=topfileName,
            ##                                         zjetfileName=zjetfileName,
            ##                                         distributionName= "mHH_l",   # this has been decided to fix on DiJetMass
            ##                                         n_trkjet  = n_trkjet,
            ##                                         n_btag    = n_btag,
            ##                                         btag_WP     = btag_WP,
            ##                                         mu_qcd_vals = bkgFitResults["muqcd"],
            ##                                         topscale_vals = bkgFitResults["topscale"],
            ##                                         NRebin = NRebin,
            ##                                         use_one_top_nuis = use_one_top_nuis,
            ##                                         use_scale_top_0b = use_scale_top_0b,
            ##                                         nbtag_top_shape_for4b = nbtag_top_shape_SRPred_for4b,
            ##                                         makePlots = True,
            ##                                         verbose = False,
            ##                                         outfileNameBase="QCDSysfit.root")

        elif inputQCDSyst_Dict != None:
            QCDSyst_Dict = inputQCDSyst_Dict

        else:
            QCDSyst_Dict = None
    else:
        QCDSyst_Dict = None

    output_Dict["QCDSystCR"] = QCDSyst_Dict
    ##################################################################

    ##### Get Signal Region Histograms ################################
    datafile = R.TFile(datafileName, "READ")
    topfile = R.TFile(topfileName, "READ")
    zjetfile = (R.TFile(zjetfileName, "READ")
                if zjetfileName != None else None)

    histos = {}

    # collect all histograms
    for r in ["44", "33", "22", "40", "30", "20"]:
        # folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP, "SR")  #folder( r[0], r[1], btag_WP)
        folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP,
                              MassRegionName)  #folder( r[0], r[1], btag_WP)

        data_r = datafile.Get(folder_r).Clone("data_" + r)
        data_r.SetDirectory(0)
        # if (r == "42") and (MassRegionName == "SR") and blindData2bSR and ( (distributionName == "DiJetMass") or (distributionName == "DiJetMassPrime") ):
        #     data_r = BlindData2bSR(data_r)

        top_r = topfile.Get(folder_r).Clone("top_" + r)
        top_r.SetDirectory(0)

        zjet_r = CheckAndGet(zjetfile, folder_r, top_r).Clone("zjet_" + r)
        zjet_r.SetDirectory(0)

        for ibin in range(1, top_r.GetNbinsX() + 1):
            if top_r.GetBinContent(ibin) < 0:
                top_r.SetBinContent(ibin, 0)
                top_r.SetBinError(ibin, 0)

        data_r.Rebin(n_rebin)
        top_r.Rebin(n_rebin)
        zjet_r.Rebin(n_rebin)

        histos[r] = {"data": data_r, "top": top_r, "zjet": zjet_r}

    datafile.Close()
    topfile.Close()
    if zjetfile != None:
        zjetfile.Close()
    ##################################################################

    ##### scaling and subtractions #################################
    for ir in range(len(regions)):
        r = regions[ir]

        output_Dict[r] = {"qcd": {}, "ttbar": {}, "zjet": {}}

        if makeOutputFiles:
            outfileStat = R.TFile("outfile_boosted_" + r + ".root", "RECREATE")

        r_0b = r[0] + "0"
        #r_3b = r[0]+"3"

        top_0b = histos[r_0b]["top"].Clone("top_0b__" + r)
        if scaleTop0b:
            top_0b.Scale((bkgFitResults["topscale"][0] if use_one_top_nuis else
                          bkgFitResults["topscale"][ir]))

        zjet_0b = histos[r_0b]["zjet"].Clone("zjet_0b__" + r)

        qcd_r = histos[r_0b]["data"].Clone("qcd__" + r)
        qcd_r.Add(
            top_0b, -1
        )  # added by Qi --- we still want top to be subtracted, given that their fraction is increasing in Run 2.
        qcd_r.Add(zjet_0b, -1)
        qcd_int = qcd_r.Integral()

        for ibin in range(1, qcd_r.GetNbinsX() + 1):
            if qcd_r.GetBinContent(ibin) < 0:
                qcd_r.SetBinContent(ibin, 0)
                qcd_r.SetBinError(ibin, 0)

        top_r = histos[r]["top"].Clone("top__" + r)
        if (nbtag_top_shape_for4b == "33") and (r == "44") and (
                MassRegionName == "SR"
        ):  # the 3b top shape is only used during the SR prediction for 44 region
            temp_scaler = top_r.Integral(
            ) / histos[nbtag_top_shape_for4b]["top"].Integral()
            top_r = histos[nbtag_top_shape_for4b]["top"].Clone("top__" + r)
            top_r.Scale(temp_scaler)
        top_int = top_r.Integral()

        zjet_r = histos[r]["zjet"].Clone("zjet__" + r)

        mu_qcd = bkgFitResults["muqcd"][ir]
        top_scale = (bkgFitResults["topscale"][0]
                     if use_one_top_nuis else bkgFitResults["topscale"][ir])

        qcd_r.Scale(mu_qcd)
        top_r.Scale(top_scale)

        # store some numbers for table later
        e_qcd = R.Double(0.0)
        e_top = R.Double(0.0)
        Nbkg_dict[r]["qcd"] = qcd_r.IntegralAndError(0,
                                                     qcd_r.GetNbinsX() + 1,
                                                     e_qcd)
        Nbkg_dict[r]["top"] = top_r.IntegralAndError(0,
                                                     top_r.GetNbinsX() + 1,
                                                     e_top)
        Nbkg_dict[r]["bkg"] = Nbkg_dict[r]["qcd"] + Nbkg_dict[r]["top"]

        Nbkg_SysList[r]["qcd"].append(float(e_qcd))
        Nbkg_SysList[r]["top"].append(float(e_top))
        Nbkg_SysList[r]["bkg"].append(
            np.sqrt(float(e_qcd)**2 + float(e_top)**2))  # Qi Question

        ## Now do smoothing ###########################################################################################
        if do_smoothing:
            ## qcd_normed = qcd_r.Clone("normed")
            ## qcd_normed.SetDirectory(0)
            ## qcd_normed.Scale(1.0 / qcd_normed.Integral())
            ## qcd_normed_sm = smoothfit.smoothfit(qcd_normed, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = True, verbose = True, outfileName="qcd_normed_smoothfit_"+r+".root")

            qcd_sm = smoothfit.smoothfit(qcd_r,
                                         fitFunction=smoothing_func,
                                         fitRange=qcdSmoothRange,
                                         makePlots=True,
                                         verbose=False,
                                         outfileName="qcd_smoothfit_" + r +
                                         ".root")
            top_sm = smoothfit.smoothfit(top_r,
                                         fitFunction=top_smoothing_func,
                                         fitRange=topSmoothRange,
                                         makePlots=True,
                                         verbose=False,
                                         outfileName="top_smoothfit_" + r +
                                         ".root")

            if addSmoothErrorBin:
                qcd_final = smoothfit.MakeSmoothHistoWithError(qcd_r, qcd_sm)
                top_final = smoothfit.MakeSmoothHistoWithError(top_r, top_sm)
            else:
                qcd_final = smoothfit.MakeSmoothHisto(qcd_r, qcd_sm["nom"])
                top_final = smoothfit.MakeSmoothHisto(top_r, top_sm["nom"])

            qcd_final.SetNameTitle("qcd_hh_" + r + "__clone",
                                   "qcd_hh_" + r + "__clone")
            top_final.SetNameTitle("ttbar_hh_" + r + "__clone",
                                   "ttbar_hh_" + r + "__clone")

        else:
            qcd_final = qcd_r.Clone("qcd_hh_" + r + "__clone")
            top_final = top_r.Clone("ttbar_hh_" + r + "__clone")

        zjet_final = zjet_r.Clone("zjet_hh_" + r + "__clone")

        if rebinFinal is not None:
            qcd_final = qcd_final.Rebin(
                len(rebinFinal) - 1,
                qcd_final.GetName() + "_rebinFinal", rebinFinal)
            top_final = top_final.Rebin(
                len(rebinFinal) - 1,
                top_final.GetName() + "_rebinFinal", rebinFinal)
            zjet_final = zjet_final.Rebin(
                len(rebinFinal) - 1,
                zjet_final.GetName() + "_rebinFinal", rebinFinal)

        if makeOutputFiles:
            outfileStat.WriteTObject(qcd_final, "qcd_hh", "Overwrite")
            outfileStat.WriteTObject(top_final, "ttbar_hh", "Overwrite")
            outfileStat.WriteTObject(zjet_final, "zjet_hh", "Overwrite")

        qcd_final.SetDirectory(0)
        top_final.SetDirectory(0)
        zjet_final.SetDirectory(0)
        output_Dict[r]["qcd"]["nom"] = qcd_final
        output_Dict[r]["ttbar"]["nom"] = top_final
        output_Dict[r]["zjet"]["nom"] = zjet_final

        # for systematics, don't need anything after this in loop
        if isSystematicVariation:
            continue

        ##################################################################################################################################
        ### propagate correlated systematics from the smoothing procedure---> these "replace" the stat error on the bins     #############
        ##################################################################################################################################
        if do_smoothing:

            ## qcd smoothing variations#################################################################
            if not addSmoothErrorBin:
                for ivar in range(len(qcd_sm["vars"])):
                    qup = qcd_sm["vars"][ivar][0]
                    qdw = qcd_sm["vars"][ivar][1]

                    qcd_r_qup = smoothfit.MakeSmoothHisto(qcd_r, qup)
                    qcd_r_qdw = smoothfit.MakeSmoothHisto(qcd_r, qdw)

                    qcd_r_qup.SetNameTitle(
                        "qcd_hh_" + r + "_smoothQ" + str(ivar) + "Up__clone",
                        "qcd_hh_" + r + "_smoothQ" + str(ivar) + "Up__clone")
                    qcd_r_qdw.SetNameTitle(
                        "qcd_hh_" + r + "_smoothQ" + str(ivar) + "Down__clone",
                        "qcd_hh_" + r + "_smoothQ" + str(ivar) + "Down__clone")

                    if rebinFinal is not None:
                        qcd_r_qup = qcd_r_qup.Rebin(
                            len(rebinFinal) - 1,
                            qcd_r_qup.GetName() + "_rebinFinal", rebinFinal)
                        qcd_r_qdw = qcd_r_qdw.Rebin(
                            len(rebinFinal) - 1,
                            qcd_r_qdw.GetName() + "_rebinFinal", rebinFinal)

                    if makeOutputFiles:
                        outfileStat.WriteTObject(
                            qcd_r_qup, "qcd_hh_smoothQ" + str(ivar) + "Up",
                            "Overwrite")
                        outfileStat.WriteTObject(
                            qcd_r_qdw, "qcd_hh_smoothQ" + str(ivar) + "Down",
                            "Overwrite")

                    qcd_r_qup.SetDirectory(0)
                    qcd_r_qdw.SetDirectory(0)
                    output_Dict[r]["qcd"]["smoothQ" + str(ivar) +
                                          "Up"] = qcd_r_qup
                    output_Dict[r]["qcd"]["smoothQ" + str(ivar) +
                                          "Down"] = qcd_r_qdw

            ## qcd smoothing function variations #################################################################
            if smoothing_func == "ExpModGauss":
                smoothFuncCompSyst = EMGSmoothSyst.smoothFuncCompare(
                    qcd_r,
                    fitFunction=smoothing_func,
                    fitRange=qcdSmoothRange,
                    funcCompareRange=(900, qcdSmoothRange[1]),
                    makePlots=True,
                    verbose=False,
                    outfileName="EMGSmoothFuncCompare_" + r + ".root",
                    plotExtra=False)  # Qi
            else:
                # smoothFuncCompSyst = smoothfit.smoothFuncCompare(qcd_r, fitRange = (900, qcdSmoothRange[1]),
                smoothFuncCompSyst = smoothfit.smoothFuncCompare(
                    qcd_r,
                    fitRange=(qcdSmoothRange[0], qcdSmoothRange[1]),  # qi
                    makePlots=True,
                    verbose=False,
                    outfileName="smoothFuncCompare_" + r + ".root",
                    plotExtra=False)  # Qi

            qcd_r_func_up = smoothFuncCompSyst["up"]
            qcd_r_func_dw = smoothFuncCompSyst["dw"]
            qcd_r_func_up_super = smoothFuncCompSyst["up_super"]
            qcd_r_func_dw_super = smoothFuncCompSyst["dw_super"]

            if rebinFinal is not None:
                qcd_r_func_up = qcd_r_func_up.Rebin(
                    len(rebinFinal) - 1,
                    qcd_r_func_up.GetName() + "_rebinFinal", rebinFinal)
                qcd_r_func_dw = qcd_r_func_dw.Rebin(
                    len(rebinFinal) - 1,
                    qcd_r_func_dw.GetName() + "_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(qcd_r_func_up, "qcd_hh_smoothFuncUp",
                                         "Overwrite")
                outfileStat.WriteTObject(qcd_r_func_dw,
                                         "qcd_hh_smoothFuncDown", "Overwrite")

                outfileStat.WriteTObject(qcd_r_func_up_super,
                                         "qcd_hh_smoothFuncSuperUp",
                                         "Overwrite")
                outfileStat.WriteTObject(qcd_r_func_dw_super,
                                         "qcd_hh_smoothFuncSuperDown",
                                         "Overwrite")

            # treat negative bin
            for ibin in range(1, qcd_r_func_up.GetNbinsX() + 1):
                if qcd_r_func_up.GetBinContent(ibin) < 0:
                    qcd_r_func_up.SetBinContent(ibin, 0)
                    qcd_r_func_up.SetBinError(ibin, 0)

                if qcd_r_func_dw.GetBinContent(ibin) < 0:
                    qcd_r_func_dw.SetBinContent(ibin, 0)
                    qcd_r_func_dw.SetBinError(ibin, 0)

                if qcd_r_func_up_super.GetBinContent(ibin) < 0:
                    qcd_r_func_up_super.SetBinContent(ibin, 0)
                    qcd_r_func_up_super.SetBinError(ibin, 0)

                if qcd_r_func_dw_super.GetBinContent(ibin) < 0:
                    qcd_r_func_dw_super.SetBinContent(ibin, 0)
                    qcd_r_func_dw_super.SetBinError(ibin, 0)

            qcd_r_func_up.SetDirectory(0)
            qcd_r_func_dw.SetDirectory(0)
            output_Dict[r]["qcd"]["smoothFuncUp"] = qcd_r_func_up
            output_Dict[r]["qcd"]["smoothFuncDown"] = qcd_r_func_dw

            qcd_r_func_up_super.SetDirectory(0)
            qcd_r_func_dw_super.SetDirectory(0)
            output_Dict[r]["qcd"]["smoothFuncUp_super"] = qcd_r_func_up_super
            output_Dict[r]["qcd"]["smoothFuncDown_super"] = qcd_r_func_dw_super

            #smoothfit.smoothFuncRangeCompare(qcd_r, fitRange = (900, qcdSmoothRange[1]), makePlots = True, verbose = False, outfileName="smoothFuncRangeCompare_"+r+".root")

            smoothfit.smoothFuncRangeCompare(
                qcd_r,
                fitFunction=smoothing_func,
                fitRange=qcdSmoothRange,
                fitMaxVals=["1750", "2000", "2500"],
                fitMinVals=[str(qcdSmoothRange[0]), "1200", "1500"],
                makePlots=True,
                plotExtra=False,
                verbose=False,
                outfileName="smoothFuncRangeCompare_" + r + ".root")  # Qi

            ## ttbar smoothing variations##############################################################################
            if not addSmoothErrorBin:
                for ivar in range(len(top_sm["vars"])):
                    tup = top_sm["vars"][ivar][0]
                    tdw = top_sm["vars"][ivar][1]

                    top_r_tup = smoothfit.MakeSmoothHisto(top_r, tup)
                    top_r_tdw = smoothfit.MakeSmoothHisto(top_r, tdw)

                    top_r_tup.SetNameTitle(
                        "ttbar_hh_" + r + "_smoothT" + str(ivar) + "Up__clone",
                        "ttbar_hh_" + r + "_smoothT" + str(ivar) + "Up__clone")
                    top_r_tdw.SetNameTitle(
                        "ttbar_hh_" + r + "_smoothT" + str(ivar) +
                        "Down__clone", "ttbar_hh_" + r + "_smoothT" +
                        str(ivar) + "Down__clone")

                    if rebinFinal is not None:
                        top_r_tup = top_r_tup.Rebin(
                            len(rebinFinal) - 1,
                            top_r_tup.GetName() + "_rebinFinal", rebinFinal)
                        top_r_tdw = top_r_tdw.Rebin(
                            len(rebinFinal) - 1,
                            top_r_tdw.GetName() + "_rebinFinal", rebinFinal)

                    if makeOutputFiles:
                        outfileStat.WriteTObject(
                            top_r_tup, "ttbar_hh_smoothT" + str(ivar) + "Up",
                            "Overwrite")
                        outfileStat.WriteTObject(
                            top_r_tdw, "ttbar_hh_smoothT" + str(ivar) + "Down",
                            "Overwrite")

                    top_r_tup.SetDirectory(0)
                    top_r_tdw.SetDirectory(0)
                    output_Dict[r]["ttbar"]["smoothT" + str(ivar) +
                                            "Up"] = top_r_tup
                    output_Dict[r]["ttbar"]["smoothT" + str(ivar) +
                                            "Down"] = top_r_tdw

        ########################################################################################################
        ### propagate correlated systematics from normalization fits for mu_qcd and top_scale    ###############
        ########################################################################################################
        for ivar in range(len(pvars)):
            sys_qcd = []
            sys_top = []
            sys_bkg = []
            for iUD in range(2):
                UpDw = ("Up" if iUD == 0 else "Down")

                mu_qcd_var = pvars[ivar][iUD][ir]
                top_scale_var = pvars[ivar][iUD][
                    n_channels + (0 if use_one_top_nuis else ir)]

                qvar = qcd_r.Clone("qvar")
                qvar.Scale(mu_qcd_var * qcd_int / qvar.Integral())

                ## for ibin in range(1, qvar.GetNbinsX()+1):
                ##     if qvar.GetBinError(ibin) > qvar.GetBinContent(ibin):
                ##         qvar.SetBinError(ibin, qvar.GetBinContent(ibin))

                tvar = top_r.Clone("tvar")
                tvar.Scale(top_scale_var * top_int / tvar.Integral())

                ### store some numbers for table
                sys_qcd.append(qvar.Integral() - Nbkg_dict[r]["qcd"])
                sys_top.append(tvar.Integral() - Nbkg_dict[r]["top"])
                sys_bkg.append(qvar.Integral() + tvar.Integral() -
                               Nbkg_dict[r]["bkg"])

                #vartxt = vartxt + str(r) + ' ' + str(ivar) + ' ' + str(iUD) + ' ' + str(qvar.Integral()) + ' ' + str(tvar.Integral()) + ' ' + str( (qvar.Integral() + tvar.Integral())) + '\n'

                ## Now do smoothing #######
                if do_smoothing:
                    qvar_sm = smoothfit.smoothfit(
                        qvar,
                        fitFunction=smoothing_func,
                        fitRange=qcdSmoothRange,
                        makePlots=False,
                        verbose=verbose,
                        outfileName="qcd_smoothfit_" + r + "_Norm" +
                        str(ivar) + str(iUD) + ".root")
                    tvar_sm = smoothfit.smoothfit(
                        tvar,
                        fitFunction=top_smoothing_func,
                        fitRange=topSmoothRange,
                        makePlots=False,
                        verbose=verbose,
                        outfileName="top_smoothfit_" + r + "_Norm" +
                        str(ivar) + str(iUD) + ".root")

                    if addSmoothErrorBin:
                        qvar_final = smoothfit.MakeSmoothHistoWithError(
                            qvar, qvar_sm)
                        tvar_final = smoothfit.MakeSmoothHistoWithError(
                            tvar, tvar_sm)
                    else:
                        qvar_final = smoothfit.MakeSmoothHisto(
                            qvar, qvar_sm["nom"])
                        tvar_final = smoothfit.MakeSmoothHisto(
                            tvar, tvar_sm["nom"])

                    qvar_final.SetNameTitle(
                        "qcd_hh_" + r + "_normY" + str(ivar) + UpDw +
                        "__clone", "qcd_hh_" + r + "_normY" + str(ivar) +
                        UpDw + "__clone")
                    tvar_final.SetNameTitle(
                        "ttbar_hh_" + r + "_normY" + str(ivar) + UpDw +
                        "__clone", "ttbar_hh_" + r + "_normY" + str(ivar) +
                        UpDw + "__clone")

                else:
                    qvar_final = qvar.Clone("qcd_hh_" + r + "_normY" +
                                            str(ivar) + UpDw + "__clone")
                    tvar_final = tvar.Clone("ttbar_hh_" + r + "_normY" +
                                            str(ivar) + UpDw + "__clone")

                if rebinFinal is not None:
                    qvar_final = qvar_final.Rebin(
                        len(rebinFinal) - 1,
                        qvar_final.GetName() + "_rebinFinal", rebinFinal)
                    tvar_final = tvar_final.Rebin(
                        len(rebinFinal) - 1,
                        tvar_final.GetName() + "_rebinFinal", rebinFinal)

                if makeOutputFiles:
                    outfileStat.WriteTObject(qvar_final,
                                             "qcd_hh_normY" + str(ivar) + UpDw,
                                             "Overwrite")
                    outfileStat.WriteTObject(
                        tvar_final, "ttbar_hh_normY" + str(ivar) + UpDw,
                        "Overwrite")

                qvar_final.SetDirectory(0)
                tvar_final.SetDirectory(0)
                output_Dict[r]["qcd"]["normY" + str(ivar) + UpDw] = qvar_final
                output_Dict[r]["ttbar"]["normY" + str(ivar) +
                                        UpDw] = tvar_final

            # store some numbers for table later
            e_qcd_i = np.max(np.abs(sys_qcd))
            e_top_i = np.max(np.abs(sys_top))
            e_bkg_i = np.max(np.abs(sys_bkg))

            Nbkg_SysList[r]["qcd"].append(e_qcd_i)
            Nbkg_SysList[r]["top"].append(e_top_i)
            Nbkg_SysList[r]["bkg"].append(e_bkg_i)

        ########################################################################################################
        ####### QCD Shape and Norm estimated from CR            ################################################
        ########################################################################################################
        if QCDSyst_Dict != None and isMhhDistribution:  # qi

            qvar_shape_up = qcd_r.Clone("qvar_QCDshape_up")
            #qvar_shape_up.Multiply( QCDSyst_Dict["Shape_"+r]["fup"] )

            qvar_shape_dw = qcd_r.Clone("qvar_QCDshape_dw")
            #qvar_shape_dw.Multiply( QCDSyst_Dict["Shape_"+r]["fdw"] )

            for ibinX in range(1, qvar_shape_up.GetNbinsX() + 1):
                if (qvar_shape_up.GetBinContent(ibinX) < 0):
                    qvar_shape_up.SetBinContent(ibinX, 0)
                    qvar_shape_up.SetBinError(ibinX, 0)

                if (qvar_shape_dw.GetBinContent(ibinX) < 0):
                    qvar_shape_dw.SetBinContent(ibinX, 0)
                    qvar_shape_dw.SetBinError(ibinX, 0)

            qvar_shape_up.Scale(qcd_r.Integral() / qvar_shape_up.Integral())
            qvar_shape_dw.Scale(qcd_r.Integral() / qvar_shape_dw.Integral())

            ## Now do smoothing
            if do_smoothing:
                qvar_shape_up_sm = smoothfit.smoothfit(
                    qvar_shape_up,
                    fitFunction=smoothing_func,
                    fitRange=qcdSmoothRange,
                    makePlots=False,
                    verbose=verbose,
                    outfileName="qcd_smoothfit_" + r + "_QCDShapeUp.root")

                qvar_shape_dw_sm = smoothfit.smoothfit(
                    qvar_shape_dw,
                    fitFunction=smoothing_func,
                    fitRange=qcdSmoothRange,
                    makePlots=False,
                    verbose=verbose,
                    outfileName="qcd_smoothfit_" + r + "_QCDShapeDown.root")

                if addSmoothErrorBin:
                    qvar_shape_up_final = smoothfit.MakeSmoothHistoWithError(
                        qvar_shape_up, qvar_shape_up_sm)
                    qvar_shape_dw_final = smoothfit.MakeSmoothHistoWithError(
                        qvar_shape_dw, qvar_shape_dw_sm)
                else:
                    qvar_shape_up_final = smoothfit.MakeSmoothHisto(
                        qvar_shape_up, qvar_shape_up_sm["nom"])
                    qvar_shape_dw_final = smoothfit.MakeSmoothHisto(
                        qvar_shape_dw, qvar_shape_dw_sm["nom"])

                qvar_shape_up_final.Multiply(QCDSyst_Dict["Shape_" + r])
                qvar_shape_dw_final.Divide(QCDSyst_Dict["Shape_" + r])

                qvar_shape_up_final.SetNameTitle(
                    "qcd_hh_" + r + "_QCDShapeCRUp__clone",
                    "qcd_hh_" + r + "_QCDShapeCRUp__clone")
                qvar_shape_dw_final.SetNameTitle(
                    "qcd_hh_" + r + "_QCDShapeCRDown__clone",
                    "qcd_hh_" + r + "_QCDShapeCRDown__clone")

            else:
                qvar_shape_up_final = qvar_shape_up.Clone(
                    "qcd_hh_" + r + "_QCDShapeCRUp__clone")
                qvar_shape_dw_final = qvar_shape_dw.Clone(
                    "qcd_hh_" + r + "_QCDShapeCRDown__clone")

            if rebinFinal is not None:
                qvar_shape_up_final = qvar_shape_up_final.Rebin(
                    len(rebinFinal) - 1,
                    qvar_shape_up_final.GetName() + "_rebinFinal", rebinFinal)
                qvar_shape_dw_final = qvar_shape_dw_final.Rebin(
                    len(rebinFinal) - 1,
                    qvar_shape_dw_final.GetName() + "_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(qvar_shape_up_final,
                                         "qcd_hh_QCDShapeCRUp")
                outfileStat.WriteTObject(qvar_shape_dw_final,
                                         "qcd_hh_QCDShapeCRDown")

            qvar_shape_up_final.SetDirectory(0)
            qvar_shape_dw_final.SetDirectory(0)
            output_Dict[r]["qcd"]["QCDShapeCRUp"] = qvar_shape_up_final
            output_Dict[r]["qcd"]["QCDShapeCRDown"] = qvar_shape_dw_final

        ###########################################################################################
        ### Norm comparison in CR      ############################################################
        ###########################################################################################
        if QCDSyst_Dict != None:

            qvar_normCR_up = qcd_final.Clone("qcd_hh_" + r +
                                             "_QCDnormCRUp__clone")
            qvar_normCR_up.Scale(1.0 + QCDSyst_Dict["Scale_" + r])

            qvar_normCR_dw = qcd_final.Clone("qcd_hh_" + r +
                                             "_QCDnormCRDown__clone")
            qvar_normCR_dw.Scale(1.0 - QCDSyst_Dict["Scale_" + r])

            if rebinFinal is not None:
                qvar_normCR_up = qvar_normCR_up.Rebin(
                    len(rebinFinal) - 1,
                    qvar_normCR_up.GetName() + "_rebinFinal", rebinFinal)
                qvar_normCR_dw = qvar_normCR_dw.Rebin(
                    len(rebinFinal) - 1,
                    qvar_normCR_dw.GetName() + "_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(qvar_normCR_up, "qcd_hh_QCDNormCRUp")
                outfileStat.WriteTObject(qvar_normCR_dw,
                                         "qcd_hh_QCDNormCRDown")

            qvar_normCR_up.SetDirectory(0)
            qvar_normCR_dw.SetDirectory(0)
            output_Dict[r]["qcd"]["QCDNormCRUp"] = qvar_normCR_up
            output_Dict[r]["qcd"]["QCDNormCRDown"] = qvar_normCR_dw

        #####################################################################################################################
        ### top shape systematics in 4b region, if using 3b shape ###########################################################
        #####################################################################################################################
        if r == "44" and nbtag_top_shape_SRPred_for4b == "33" and MassRegionName == "SR" and isMhhDistribution:  # qi
            ## ttbarShapeSRSyst_Dict = SystTools.ttbarShapeSysSR(topfileName,
            ##                                                     distributionName,
            ##                                                     signal_region = "22",
            ##                                                     compare_region = "33",
            ##                                                     btag_WP     = btag_WP,
            ##                                                     makePlots = True,
            ##                                                     verbose = False,
            ##                                                     outfileNameBase="TopShapeSRSysfit.root")

            ttbarShapeSRSyst_Dict = SystToolsSmooth.ttbarShapeSysSR(
                topfileName,
                distributionName,
                signal_region="33",
                compare_region="22",
                btag_WP=btag_WP,
                smoothing_func=top_smoothing_func,
                SmoothRange=topSmoothRange,  # (100, 2500),
                makePlots=True,
                verbose=False,
                outfileNameBase="TopShapeSRSysfitSmooth.root")

            tvar_shape_up = top_r.Clone("tvar_ttbarShapeSR_up")
            #tvar_shape_up.Multiply( ttbarShapeSRSyst_Dict["fup"] )

            tvar_shape_dw = top_r.Clone("tvar_ttbarShapeSR_dw")
            #tvar_shape_dw.Multiply( ttbarShapeSRSyst_Dict["fdw"] )

            for ibinX in range(1, tvar_shape_up.GetNbinsX() + 1):
                if (tvar_shape_up.GetBinContent(ibinX) < 0):
                    tvar_shape_up.SetBinContent(ibinX, 0)
                    tvar_shape_up.SetBinError(ibinX, 0)

                if (tvar_shape_dw.GetBinContent(ibinX) < 0):
                    tvar_shape_dw.SetBinContent(ibinX, 0)
                    tvar_shape_dw.SetBinError(ibinX, 0)

            tvar_shape_up.Scale(top_r.Integral() / tvar_shape_up.Integral())
            tvar_shape_dw.Scale(top_r.Integral() / tvar_shape_dw.Integral())

            ## Now do smoothing ##########################
            if do_smoothing:
                tvar_shape_up_sm = smoothfit.smoothfit(
                    tvar_shape_up,
                    fitFunction=top_smoothing_func,
                    fitRange=topSmoothRange,
                    makePlots=False,
                    verbose=verbose,
                    outfileName="top_smoothfit_" + r + "_ttbarShapeSRUp.root")

                tvar_shape_dw_sm = smoothfit.smoothfit(
                    tvar_shape_dw,
                    fitFunction=top_smoothing_func,
                    fitRange=topSmoothRange,
                    makePlots=False,
                    verbose=verbose,
                    outfileName="top_smoothfit_" + r +
                    "_ttbarShapeSReDown.root")

                if addSmoothErrorBin:
                    tvar_shape_up_final = smoothfit.MakeSmoothHistoWithError(
                        tvar_shape_up, tvar_shape_up_sm)
                    tvar_shape_dw_final = smoothfit.MakeSmoothHistoWithError(
                        tvar_shape_dw, tvar_shape_dw_sm)
                else:
                    tvar_shape_up_final = smoothfit.MakeSmoothHisto(
                        tvar_shape_up, tvar_shape_up_sm["nom"])
                    tvar_shape_dw_final = smoothfit.MakeSmoothHisto(
                        tvar_shape_dw, tvar_shape_dw_sm["nom"])

                tvar_shape_up_final.Multiply(ttbarShapeSRSyst_Dict["Shape"])
                tvar_shape_dw_final.Divide(ttbarShapeSRSyst_Dict["Shape"])

                tvar_shape_up_final.SetNameTitle(
                    "ttbar_hh_" + r + "_ttbarShapeSRUp__clone",
                    "ttbar_hh_" + r + "_ttbarShapeSRUp__clone")
                tvar_shape_dw_final.SetNameTitle(
                    "ttbar_hh_" + r + "_ttbarShapeSRDown__clone",
                    "ttbar_hh_" + r + "_ttbarShapeSRDown__clone")

            else:
                tvar_shape_up_final = tvar_shape_up.Clone(
                    "ttbar_hh_" + r + "_ttbarShapeSRUp__clone")
                tvar_shape_dw_final = tvar_shape_dw.Clone(
                    "ttbar_hh_" + r + "_ttbarShapeSRDown__clone")

            if rebinFinal is not None:
                tvar_shape_up_final = tvar_shape_up_final.Rebin(
                    len(rebinFinal) - 1,
                    tvar_shape_up_final.GetName() + "_rebinFinal", rebinFinal)
                tvar_shape_dw_final = tvar_shape_dw_final.Rebin(
                    len(rebinFinal) - 1,
                    tvar_shape_dw_final.GetName() + "_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(tvar_shape_up_final,
                                         "ttbar_hh_ttbarShapeSRUp")
                outfileStat.WriteTObject(tvar_shape_dw_final,
                                         "ttbar_hh_ttbarShapeSRDown")

            tvar_shape_up_final.SetDirectory(0)
            tvar_shape_dw_final.SetDirectory(0)
            output_Dict[r]["ttbar"]["ttbarShapeSRUp"] = tvar_shape_up_final
            output_Dict[r]["ttbar"]["ttbarShapeSRDown"] = tvar_shape_dw_final

        ### close outfiles, if used ###
        if makeOutputFiles:
            outfileStat.Close()

    ### Print tables ###
    #PrintTable( Nbkg_dict, Nbkg_SysList, regions)
    #print vartxt

    #print output_Dict

    output_Dict['regions'] = regions

    return output_Dict
예제 #6
0
def HistoAnalysis(datafileName="/afs/cern.ch/user/b/btong/work/bbbb/MoriondAnalysis/Output/Moriond/data_test/hist-MiniNTuple.root",
                  topfileName="/afs/cern.ch/user/b/btong/work/bbbb/MoriondAnalysis/Output/Moriond/ttbar_comb_test/hist-MiniNTuple.root",
                  zjetfileName="/afs/cern.ch/user/b/btong/work/bbbb/MoriondAnalysis/Output/Moriond/zjets_test/hist-MiniNTuple.root",
                  distributionName= "mHH_l",
                  n_trkjet  = ["4","3","2"],
                  n_btag    = ["4","3","2"],
                  btag_WP   = "70",
                  NRebin    = 20,
                  use_one_top_nuis = False,
                  use_scale_top_0b = False,
                  nbtag_top_shape_SRPred_for4b = "33",
                  rebinFinal = None,
                  smoothing_func = "Dijet",
                  top_smoothing_func = "Dijet",
                  inputFitResult = None,
                  inputQCDSyst_Dict = None,
                  doSmoothing = True,
                  addSmoothErrorBin = False,
                  qcdSmoothRange = (1200, 3000), #(1200, 3000),
                  topSmoothRange = (1200, 3000), #(1200, 3000),
                  isSystematicVariation = False,
                  verbose = False,
                  makeOutputFiles = True,
                  MassRegionName = "SR",
                  do_variable_rebin = False
                  ):
    ##### Parse Inputs ############################################
    fitzjets    = False
    dist_name   = distributionName
    print "the chosen hist is: ", dist_name
    if "pole" in distributionName:#change the smoothing range if pole distributions
        qcdSmoothRange = (1200, 3000)
        topSmoothRange = (1200, 3000)
    
    num_trkjet  = np.asarray(n_trkjet)
    if num_trkjet.shape==():
        num_trkjet = np.asarray([n_trkjet])

    num_btag    = np.asarray(n_btag)
    if num_btag.shape==():
        num_btag = np.asarray([n_btag])
    if num_btag.shape!=num_trkjet.shape:
        print "Must have same number of track jet and b-tag regions specified"
        sys.exit(0)
  
    btag_WP     = btag_WP
    
    n_rebin     = NRebin

    nbtag_top_shape_for4b = nbtag_top_shape_SRPred_for4b
    topShape_nbtag_for4b = nbtag_top_shape_for4b
    if nbtag_top_shape_for4b == None:
        topShape_nbtag_for4b = num_btag+num_btag

    useOneTopNuis = use_one_top_nuis

    scaleTop0b = use_scale_top_0b

    n_channels = num_trkjet.shape[0]

    regions = [ num_trkjet[i]+num_btag[i] for i in range(n_channels) ]

    ##for outputing
    isMhhDistribution = (distributionName=="mHH_l" or distributionName=="mHH_pole")
    do_smoothing  = (doSmoothing if isMhhDistribution else False)   # qi
    ##################################################################
    ##### Storage Variables ############################################
    output_Dict = { }
    Nbkg_dict    = {  }
    Nbkg_SysList = {  }
    for ir in regions:
        Nbkg_dict[ir]    = { "qcd":0,  "top":0,  "zjet":0,  "bkg":0, "data":0 }
        Nbkg_SysList[ir] = { "qcd":[], "top":[], "zjet":[], "bkg":[], "data":[] }
    vartxt = ''
    ##################################################################    
    ##### Do Background Fits ############################################
    ##### This is just the same fitting procedure
    if inputFitResult == None:
        bkgFitResults = BkgFit.BackgroundFit(datafileName=datafileName,
                                              topfileName=topfileName,
                                              zjetfileName=zjetfileName,
                                              distributionName = ["leadHCand_Mass"],
                                              whichFunc = "XhhBoosted",
                                              n_trkjet  = n_trkjet,
                                              n_btag    = n_btag,
                                              btag_WP   = btag_WP,
                                              NRebin    = 2,#NRebin, #this is reset to be fine binned
                                              use_one_top_nuis = use_one_top_nuis,
                                              makePlots = True,
                                              BKG_lst   = ["FourTag", "ThreeTag", "TwoTag_split"],
                                              BKG_dic   = {"FourTag":"NoTag_4Trk",  "ThreeTag":"NoTag_3Trk", "TwoTag_split":"NoTag_2Trk_split",  "TwoTag":"OneTag",  "OneTag":"NoTag"},
                                              fitzjets  = fitzjets)

    else:
        bkgFitResults = inputFitResult

    pvars = bkgFitResults["pvars"]
    output_Dict["fitResults"] = bkgFitResults
    ##################################################################
    ##### Get QCD Shape Systematics from CR  ##############################
    print "STEP: Get QCD Shape Systematics from CR"
    ##### This is smoothing the CR region distributions
    if MassRegionName == "SR":
        # should only affect SR
        if inputQCDSyst_Dict == None and isMhhDistribution:  # qi
            #or this option QCDSyst_Dict = SystTools.QCDSystematics(datafileName=datafileName
            QCDSyst_Dict =  SystToolsSmooth.QCDSystematics(datafileName=datafileName,
                                        topfileName=topfileName,
                                        zjetfileName=zjetfileName,
                                        distributionName= "mHH_l",   # this has been decided to fix on DiJetMass
                                        n_trkjet  = n_trkjet,
                                        n_btag    = n_btag,
                                        btag_WP     = btag_WP,
                                        mu_qcd_vals = bkgFitResults["muqcd"],
                                        topscale_vals = bkgFitResults["muttbar"],
                                        NRebin = 5, #this used to be 5, incease to 10 just like SR
                                        smoothing_func = smoothing_func,
                                        SmoothRange = (1100, 3000),# (100, 2500), #this is fixed...
                                        use_one_top_nuis = use_one_top_nuis,
                                        use_scale_top_0b = use_scale_top_0b,
                                        nbtag_top_shape_for4b = nbtag_top_shape_SRPred_for4b,
                                        makePlots = True,
                                        verbose = False,
                                        outfileNameBase="QCDSysfitSmooth.root") 
            
        elif inputQCDSyst_Dict != None:
            QCDSyst_Dict = inputQCDSyst_Dict

        else:
            QCDSyst_Dict = None
    else:
        QCDSyst_Dict = None

    output_Dict["QCDSystCR"] = QCDSyst_Dict

    ##################################################################
    ##### Get Signal Region Histograms ################################
    print "STEP: Get Signal Region Histograms"
    ##### This is loding input file histograms
    datafile  = R.TFile(datafileName,"READ")
    topfile   = R.TFile(topfileName,"READ")
    zjetfile  = ( R.TFile(zjetfileName,"READ") if fitzjets is True else None)
    histos    = {}
    
    # collect all histograms
    for r in ["44","33","22","40","30","20"]:
        folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP, MassRegionName)  #folder( r[0], r[1], btag_WP)
        
        data_r   = datafile.Get(folder_r).Clone("data_"+r)
        data_r.SetDirectory(0)

        top_r    = topfile.Get(folder_r).Clone("top_"+r)
        top_r.SetDirectory(0)

        zjet_r   = CheckAndGet(zjetfile, folder_r, top_r).Clone("zjet_"+r)
        zjet_r.SetDirectory(0)

        #clear the negative weight bins for ttbar
        ClearNegBin(top_r)
        
        if do_variable_rebin:
            data_r = smoothfit_Ultimate.VariableRebin(data_r,5,2000).Clone()
            top_r = smoothfit_Ultimate.VariableRebin(top_r,5,2000).Clone()
            zjet_r = smoothfit_Ultimate.VariableRebin(zjet_r,5,2000).Clone()
        else:
            data_r.Rebin(n_rebin)
            top_r.Rebin(n_rebin)
            zjet_r.Rebin(n_rebin)

        histos[r] = {"data": data_r,  "top": top_r,  "zjet":zjet_r}

    datafile.Close()
    topfile.Close()
    if zjetfile != None:
        zjetfile.Close()

    ##################################################################
    ##### scaling and subtractions #################################
    print "STEP: scaling and subtractions"
    ##### This is loding input file histograms
    for ir in range(len(regions)):
        # print ir
        r = regions[ir]

        output_Dict[r] = {"qcd":{}, "ttbar":{}, "zjet":{}}
        
        if makeOutputFiles:
            cut_lst = {"44":"FourTag", "33":"ThreeTag", "22":"TwoTag_split"}
            outfileStat = R.TFile("outfile_boosted_"+cut_lst[r]+".root","RECREATE")
        
        r_0b = r[0]+"0"
        #r_3b = r[0]+"3"
        top_0b = histos[r_0b]["top"].Clone("top_0b__"+r)
        if scaleTop0b:
            top_0b.Scale( (bkgFitResults["muttbar"][0] if use_one_top_nuis else bkgFitResults["muttbar"][ir]) )

        zjet_0b = histos[r_0b]["zjet"].Clone("zjet_0b__"+r)

        qcd_r = histos[r_0b]["data"].Clone("qcd__"+r)
        qcd_r.Add( top_0b, -1)
        qcd_r.Add( zjet_0b, -1)
        qcd_int = qcd_r.Integral()

        #clear the negative weight bins for qcd as well
        ClearNegBin(qcd_r)

        top_r = histos[r]["top"].Clone("top__"+r)
        if (nbtag_top_shape_for4b == "33") and (r == "44") and (MassRegionName == "SR"):   # the 3b top shape is only used during the SR prediction for 44 region
            temp_scaler = top_r.Integral() / histos[nbtag_top_shape_for4b]["top"].Integral()
            top_r = histos[nbtag_top_shape_for4b]["top"].Clone("top__"+r)
            top_r.Scale( temp_scaler )
        top_int = top_r.Integral()
        #print top_r.Integral(), "here! 1"

        zjet_r = histos[r]["zjet"].Clone("zjet__"+r)

        mu_qcd = bkgFitResults["muqcd"][ir]
        top_scale = (bkgFitResults["muttbar"][0] if use_one_top_nuis else bkgFitResults["muttbar"][ir])
        
        qcd_r.Scale( mu_qcd )
        top_r.Scale( top_scale )
        print "top total:", top_r.Integral(), " ; qcd total:", qcd_r.Integral(), "here! 2"


        bkg_r = qcd_r.Clone("bkg__" + r)
        bkg_r.Add( top_r, 1)
        bkg_r.Add( zjet_r, 1)
        # store some numbers for the output table later
        e_qcd = R.Double(0.0)
        e_top = R.Double(0.0)
        e_bkg = R.Double(0.0)
        e_data = R.Double(0.0)
        Nbkg_dict[r]["qcd"] = qcd_r.IntegralAndError(0, qcd_r.GetNbinsX()+1, e_qcd)
        Nbkg_dict[r]["top"] = top_r.IntegralAndError(0, top_r.GetNbinsX()+1, e_top)
        Nbkg_dict[r]["bkg"] = bkg_r.IntegralAndError(0, bkg_r.GetNbinsX()+1, e_bkg)
        Nbkg_dict[r]["data"] = histos[r]["data"].IntegralAndError(0, histos[r]["data"].GetNbinsX()+1, e_data)

        Nbkg_SysList[r]["qcd"].append( float(e_qcd) )
        Nbkg_SysList[r]["top"].append( float(e_top) )
        Nbkg_SysList[r]["bkg"].append( float(e_bkg) )   # Qi Question; Tony Question as well...
        Nbkg_SysList[r]["data"].append( float(e_data) )
        
        ## Now do smoothing ###########################################################################################
        print "start smoothing: ", ir
        if do_smoothing:
            qcd_sm = smoothfit.smoothfit(qcd_r, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = True, verbose = False, outfileName="qcd_smoothfit_"+r+".root")
            top_sm = smoothfit.smoothfit(top_r, fitFunction = top_smoothing_func, fitRange = topSmoothRange, makePlots = True, verbose = False, outfileName="top_smoothfit_"+r+".root")
            print "top total:", top_r.Integral(), " ; qcd total:", qcd_r.Integral(), "here! 2.5"
            if addSmoothErrorBin:
                qcd_final = smoothfit.MakeSmoothHistoWithError(qcd_r, qcd_sm)
                top_final = smoothfit.MakeSmoothHistoWithError(top_r, top_sm)
            else:
                qcd_final = smoothfit.MakeSmoothHisto(qcd_r, qcd_sm["nom"])
                top_final = smoothfit.MakeSmoothHisto(top_r, top_sm["nom"])
            qcd_final.SetNameTitle("qcd_hh_"+r+"__clone",   "qcd_hh_"+r+"__clone")
            top_final.SetNameTitle("ttbar_hh_"+r+"__clone", "ttbar_hh_"+r+"__clone")
        else:
            qcd_final = qcd_r.Clone("qcd_hh_"+r+"__clone")
            top_final = top_r.Clone("ttbar_hh_"+r+"__clone")
        
        print "top total:", top_final.Integral(), " ; qcd total:", qcd_final.Integral(), "here! 3"
        zjet_final = zjet_r.Clone("zjet_hh_"+r+"__clone")
        
        if rebinFinal is not None:
            qcd_final = qcd_final.Rebin(len(rebinFinal)-1, qcd_final.GetName()+"_rebinFinal", rebinFinal)
            top_final = top_final.Rebin(len(rebinFinal)-1, top_final.GetName()+"_rebinFinal", rebinFinal)
            zjet_final = zjet_final.Rebin(len(rebinFinal)-1, zjet_final.GetName()+"_rebinFinal", rebinFinal)

        if makeOutputFiles:
            outfileStat.WriteTObject(qcd_final, "qcd_hh","Overwrite")
            outfileStat.WriteTObject(top_final, "ttbar_hh","Overwrite")
            outfileStat.WriteTObject(zjet_final, "zjet_hh","Overwrite")

        qcd_final.SetDirectory(0)
        top_final.SetDirectory(0)
        zjet_final.SetDirectory(0)
        output_Dict[r]["qcd"]["nom"] = qcd_final
        output_Dict[r]["ttbar"]["nom"] = top_final
        output_Dict[r]["zjet"]["nom"] = zjet_final

        # for systematics, don't need anything after this in loop
        if isSystematicVariation:
            continue

        ##################################################################################################################################
        ### propagate correlated systematics from the smoothing procedure---> these "replace" the stat error on the bins     #############
        ##################################################################################################################################
        ##### This is adding smoothing systematics
        if do_smoothing:
            ## qcd smoothing variations#################################################################
            if not addSmoothErrorBin:
                for ivar in range(len(qcd_sm["vars"])):
                    qup = qcd_sm["vars"][ivar][0]
                    qdw = qcd_sm["vars"][ivar][1]

                    qcd_r_qup = smoothfit.MakeSmoothHisto(qcd_r, qup)
                    qcd_r_qdw = smoothfit.MakeSmoothHisto(qcd_r, qdw)

                    qcd_r_qup.SetNameTitle("qcd_hh_"+r+"_smoothQ"+str(ivar)+"up__clone", "qcd_hh_"+r+"_smoothQ"+str(ivar)+"up__clone")
                    qcd_r_qdw.SetNameTitle("qcd_hh_"+r+"_smoothQ"+str(ivar)+"down__clone", "qcd_hh_"+r+"_smoothQ"+str(ivar)+"down__clone")

                    if rebinFinal is not None:
                        qcd_r_qup = qcd_r_qup.Rebin(len(rebinFinal)-1, qcd_r_qup.GetName()+"_rebinFinal", rebinFinal)
                        qcd_r_qdw = qcd_r_qdw.Rebin(len(rebinFinal)-1, qcd_r_qdw.GetName()+"_rebinFinal", rebinFinal)

                    if makeOutputFiles:
                        outfileStat.WriteTObject(qcd_r_qup, "qcd_hh_smoothQ"+str(ivar)+"up","Overwrite")
                        outfileStat.WriteTObject(qcd_r_qdw, "qcd_hh_smoothQ"+str(ivar)+"down","Overwrite")

                    qcd_r_qup.SetDirectory(0)
                    qcd_r_qdw.SetDirectory(0)
                    output_Dict[r]["qcd"]["smoothQ"+str(ivar)+"up"] = qcd_r_qup
                    output_Dict[r]["qcd"]["smoothQ"+str(ivar)+"down"] = qcd_r_qdw
   
            ## qcd smoothing function variations #################################################################
            if smoothing_func == "ExpModGauss":
                smoothFuncCompSyst = EMGSmoothSyst.smoothFuncCompare(qcd_r, fitFunction = smoothing_func,
                                                                     fitRange = qcdSmoothRange, funcCompareRange=(900, qcdSmoothRange[1]),
                                                                     makePlots = True, verbose = False, outfileName="EMGSmoothFuncCompare_"+r+".root", plotExtra=False)  # Qi
            else:
                # smoothFuncCompSyst = smoothfit.smoothFuncCompare(qcd_r, fitRange = (900, qcdSmoothRange[1]),
                smoothFuncCompSyst = smoothfit.smoothFuncCompare(qcd_r, fitRange = qcdSmoothRange,            # qi
                                                                 makePlots = True, verbose = False, outfileName="smoothFuncCompare_"+r+".root", plotExtra=False)  # Qi
                
            qcd_r_func_up = smoothFuncCompSyst["up"]
            qcd_r_func_dw = smoothFuncCompSyst["dw"]
            qcd_r_func_up_super = smoothFuncCompSyst["up_super"]
            qcd_r_func_dw_super = smoothFuncCompSyst["dw_super"]

            if rebinFinal is not None:
                qcd_r_func_up = qcd_r_func_up.Rebin(len(rebinFinal)-1, qcd_r_func_up.GetName()+"_rebinFinal", rebinFinal)
                qcd_r_func_dw = qcd_r_func_dw.Rebin(len(rebinFinal)-1, qcd_r_func_dw.GetName()+"_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(qcd_r_func_up, "qcd_hh_smoothFuncup","Overwrite")
                outfileStat.WriteTObject(qcd_r_func_dw, "qcd_hh_smoothFuncdown","Overwrite")
                
                outfileStat.WriteTObject(qcd_r_func_up_super, "qcd_hh_smoothFuncSuperup","Overwrite")
                outfileStat.WriteTObject(qcd_r_func_dw_super, "qcd_hh_smoothFuncSuperdown","Overwrite")

            # treat negative bin
            ClearNegBin(qcd_r_func_up)
            ClearNegBin(qcd_r_func_dw)
            ClearNegBin(qcd_r_func_up_super)
            ClearNegBin(qcd_r_func_dw_super)

            qcd_r_func_up.SetDirectory(0)
            qcd_r_func_dw.SetDirectory(0)
            output_Dict[r]["qcd"]["smoothFuncup"] = qcd_r_func_up
            output_Dict[r]["qcd"]["smoothFuncdown"] = qcd_r_func_dw

            qcd_r_func_up_super.SetDirectory(0)
            qcd_r_func_dw_super.SetDirectory(0)
            output_Dict[r]["qcd"]["smoothFuncup_super"] = qcd_r_func_up_super
            output_Dict[r]["qcd"]["smoothFuncdown_super"] = qcd_r_func_dw_super
            
            stepped_min_vals = []
            stepped_max_vals = []
            stepped_fitting = True
            if stepped_fitting == True:
                starting_bin = qcd_r_qup.FindBin(qcdSmoothRange[0])
                stepped_min_vals.append(str(qcdSmoothRange[0]))
                for step in range(0, 4):
                    current_starting_bin = starting_bin + step*1
                    current_starting_mass = qcd_r_qup.GetBinCenter(current_starting_bin)
                    stepped_min_vals.append(str(current_starting_mass))
                    
                stepped_max_vals = ["3000"]
            else:
                stepped_max_vals = ["1850","2000","2250","2500"]
                stepped_min_vals = [str(qcdSmoothRange[0]),"1300","1400"]    
            print "MAX AND MIN ARE:"
            print stepped_max_vals
            print stepped_min_vals
            
            smoothfit.smoothFuncRangeCompare(qcd_r, fitFunction = smoothing_func, fitRange = qcdSmoothRange, fitMaxVals = stepped_max_vals, fitMinVals=stepped_min_vals,
                                            makePlots = True, plotExtra = False, verbose = False, outfileName="smoothFuncRangeCompare_"+r+".root")   # Qi
            
            ## ttbar smoothing variations##############################################################################
            if not addSmoothErrorBin:
                for ivar in range(len(top_sm["vars"])):
                    tup = top_sm["vars"][ivar][0]
                    tdw = top_sm["vars"][ivar][1]

                    top_r_tup = smoothfit.MakeSmoothHisto(top_r, tup)
                    top_r_tdw = smoothfit.MakeSmoothHisto(top_r, tdw)

                    top_r_tup.SetNameTitle("ttbar_hh_"+r+"_smoothQ"+str(ivar)+"up__clone",   "ttbar_hh_"+r+"_smoothQ"+str(ivar)+"up__clone")
                    top_r_tdw.SetNameTitle("ttbar_hh_"+r+"_smoothQ"+str(ivar)+"down__clone", "ttbar_hh_"+r+"_smoothQ"+str(ivar)+"down__clone")

                    if rebinFinal is not None:
                        top_r_tup = top_r_tup.Rebin(len(rebinFinal)-1, top_r_tup.GetName()+"_rebinFinal", rebinFinal)
                        top_r_tdw = top_r_tdw.Rebin(len(rebinFinal)-1, top_r_tdw.GetName()+"_rebinFinal", rebinFinal)

                    if makeOutputFiles:
                        outfileStat.WriteTObject(top_r_tup, "ttbar_hh_smoothQ"+str(ivar)+"up","Overwrite")
                        outfileStat.WriteTObject(top_r_tdw, "ttbar_hh_smoothQ"+str(ivar)+"down","Overwrite")

                    ClearNegBin(top_r_tup)
                    ClearNegBin(top_r_tdw)
                    top_r_tup.SetDirectory(0)
                    top_r_tdw.SetDirectory(0)
                    output_Dict[r]["ttbar"]["smoothQ"+str(ivar)+"up"] = top_r_tup
                    output_Dict[r]["ttbar"]["smoothQ"+str(ivar)+"down"] = top_r_tdw


        ########################################################################################################
        ### propagate correlated systematics from normalization fits for mu_qcd and top_scale    ###############
        ########################################################################################################
        ##### This is adding systematics from the fit
        #print pvars
        for ivar in range(len(pvars)):
            sys_qcd = []
            sys_top = []
            sys_bkg = []
            for iUD in range(2):
                upDw = ("up" if iUD ==0 else "down")

                mu_qcd_var = pvars[ivar][iUD][ir]
                top_scale_var = pvars[ivar][iUD][n_channels + (0 if use_one_top_nuis else ir) ]

                qvar = qcd_r.Clone("qvar")
                qvar.Scale( mu_qcd_var * qcd_int / qvar.Integral() )

                ## for ibin in range(1, qvar.GetNbinsX()+1):
                ##     if qvar.GetBinError(ibin) > qvar.GetBinContent(ibin):
                ##         qvar.SetBinError(ibin, qvar.GetBinContent(ibin))
                tvar = top_r.Clone("tvar")
                tvar.Scale( top_scale_var * top_int / tvar.Integral() )

                ### store some numbers for table
                sys_qcd.append( qvar.Integral() - Nbkg_dict[r]["qcd"] )
                sys_top.append( tvar.Integral() - Nbkg_dict[r]["top"] )
                sys_bkg.append( qvar.Integral() + tvar.Integral() - Nbkg_dict[r]["bkg"] )

                #vartxt = vartxt + str(r) + ' ' + str(ivar) + ' ' + str(iUD) + ' ' + str(qvar.Integral()) + ' ' + str(tvar.Integral()) + ' ' + str( (qvar.Integral() + tvar.Integral())) + '\n'
                ## Now do smoothing #######
                if do_smoothing:
                    qvar_sm = smoothfit.smoothfit(qvar, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = False, verbose = verbose,
                                                  outfileName="qcd_smoothfit_"+r+"_Norm"+str(ivar)+str(iUD)+".root")
                    tvar_sm = smoothfit.smoothfit(tvar, fitFunction = top_smoothing_func, fitRange = topSmoothRange, makePlots = False, verbose = verbose,
                                                  outfileName="top_smoothfit_"+r+"_Norm"+str(ivar)+str(iUD)+".root")

                    if addSmoothErrorBin:
                        qvar_final = smoothfit.MakeSmoothHistoWithError(qvar, qvar_sm)
                        tvar_final = smoothfit.MakeSmoothHistoWithError(tvar, tvar_sm)
                    else:
                        qvar_final = smoothfit.MakeSmoothHisto(qvar, qvar_sm["nom"])
                        tvar_final = smoothfit.MakeSmoothHisto(tvar, tvar_sm["nom"])

                    qvar_final.SetNameTitle("qcd_hh_"+r+"_normY"+str(ivar)+upDw+"__clone",   "qcd_hh_"+r+"_normY"+str(ivar)+upDw+"__clone")
                    tvar_final.SetNameTitle("ttbar_hh_"+r+"_normY"+str(ivar)+upDw+"__clone", "ttbar_hh_"+r+"_normY"+str(ivar)+upDw+"__clone")

                else:
                    qvar_final = qvar.Clone("qcd_hh_"+r+"_normY"+str(ivar)+upDw+"__clone")
                    tvar_final = tvar.Clone("ttbar_hh_"+r+"_normY"+str(ivar)+upDw+"__clone")

                if rebinFinal is not None:
                    qvar_final = qvar_final.Rebin(len(rebinFinal)-1, qvar_final.GetName()+"_rebinFinal", rebinFinal)
                    tvar_final = tvar_final.Rebin(len(rebinFinal)-1, tvar_final.GetName()+"_rebinFinal", rebinFinal)

                if makeOutputFiles:
                    outfileStat.WriteTObject(qvar_final, "qcd_hh_normY"+str(ivar)+upDw,"Overwrite")
                    outfileStat.WriteTObject(tvar_final, "ttbar_hh_normY"+str(ivar)+upDw,"Overwrite")

                qvar_final.SetDirectory(0)
                tvar_final.SetDirectory(0)
                output_Dict[r]["qcd"]["normY"+str(ivar)+upDw] = qvar_final
                output_Dict[r]["ttbar"]["normY"+str(ivar)+upDw] = tvar_final

                
            # store some numbers for table later
            e_qcd_i = np.max( np.abs(sys_qcd) )
            e_top_i = np.max( np.abs(sys_top) )
            e_bkg_i = np.max( np.abs(sys_bkg) )
            Nbkg_SysList[r]["qcd"].append( e_qcd_i )
            Nbkg_SysList[r]["top"].append( e_top_i )
            Nbkg_SysList[r]["bkg"].append( e_bkg_i )


        ########################################################################################################
        ####### QCD Shape and Norm estimated from CR            ################################################
        ########################################################################################################
        ##### This is adding systematics from the CR
        if QCDSyst_Dict!=None and isMhhDistribution:  # qi
            original_norm = qcd_r.Integral()
            qvar_shape_up = qcd_r.Clone("qvar_QCDshape_up")
            qvar_shape_dw = qcd_r.Clone("qvar_QCDshape_dw")

            ClearNegBin(qvar_shape_up)
            ClearNegBin(qvar_shape_dw)
        
            ## Now do smoothing
            if do_smoothing:
                qvar_shape_up_sm = smoothfit.smoothfit(qvar_shape_up, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = False, verbose = verbose,
                                                        outfileName="qcd_smoothfit_"+r+"_QCDShapeup.root")
                qvar_shape_dw_sm = smoothfit.smoothfit(qvar_shape_dw, fitFunction = smoothing_func, fitRange = qcdSmoothRange, makePlots = False, verbose = verbose,
                                                        outfileName="qcd_smoothfit_"+r+"_QCDShapedown.root")

                if addSmoothErrorBin:
                    qvar_shape_up_final = smoothfit.MakeSmoothHistoWithError(qvar_shape_up, qvar_shape_up_sm)
                    qvar_shape_dw_final = smoothfit.MakeSmoothHistoWithError(qvar_shape_dw, qvar_shape_dw_sm)
                else:
                    qvar_shape_up_final = smoothfit.MakeSmoothHisto(qvar_shape_up, qvar_shape_up_sm["nom"])
                    qvar_shape_dw_final = smoothfit.MakeSmoothHisto(qvar_shape_dw, qvar_shape_dw_sm["nom"])

                qvar_shape_up_final.Multiply( QCDSyst_Dict["Shape_"+r] )
                qvar_shape_dw_final.Divide( QCDSyst_Dict["Shape_"+r] )

                qvar_shape_up_final.SetNameTitle("qcd_hh_"+r+"_QCDShapeCRup__clone",     "qcd_hh_"+r+"_QCDShapeCRup__clone")
                qvar_shape_dw_final.SetNameTitle("qcd_hh_"+r+"_QCDShapeCRdown__clone",   "qcd_hh_"+r+"_QCDShapeCRdown__clone")
            else:
                qvar_shape_up_final = qvar_shape_up.Clone("qcd_hh_"+r+"_QCDShapeCRup__clone")
                qvar_shape_dw_final = qvar_shape_dw.Clone("qcd_hh_"+r+"_QCDShapeCRdown__clone")

            #make sure normalization is correct!
            qvar_shape_up_final.Scale( original_norm/qvar_shape_up_final.Integral() )
            qvar_shape_dw_final.Scale( original_norm/qvar_shape_dw_final.Integral() )
            if rebinFinal is not None:
                qvar_shape_up_final = qvar_shape_up_final.Rebin(len(rebinFinal)-1, qvar_shape_up_final.GetName()+"_rebinFinal", rebinFinal)
                qvar_shape_dw_final = qvar_shape_dw_final.Rebin(len(rebinFinal)-1, qvar_shape_dw_final.GetName()+"_rebinFinal", rebinFinal)

            if makeOutputFiles:

                outfileStat.WriteTObject(qvar_shape_up_final, "qcd_hh_QCDShapeCRup")
                outfileStat.WriteTObject(qvar_shape_dw_final, "qcd_hh_QCDShapeCRdown")

            qvar_shape_up_final.SetDirectory(0)
            qvar_shape_dw_final.SetDirectory(0)
            output_Dict[r]["qcd"]["QCDShapeCRup"] = qvar_shape_up_final
            output_Dict[r]["qcd"]["QCDShapeCRdown"] = qvar_shape_dw_final

        ###########################################################################################
        ### Norm comparison in CR      ############################################################
        ###########################################################################################
        if QCDSyst_Dict != None:
            
            qvar_normCR_up =  qcd_final.Clone("qcd_hh_"+r+"_QCDnormCRup__clone")
            qvar_normCR_up.Scale( 1.0 + QCDSyst_Dict["Scale_"+r] )

            qvar_normCR_dw =  qcd_final.Clone("qcd_hh_"+r+"_QCDnormCRdown__clone")
            qvar_normCR_dw.Scale( 1.0 - QCDSyst_Dict["Scale_"+r] )

            if rebinFinal is not None:
                qvar_normCR_up = qvar_normCR_up.Rebin(len(rebinFinal)-1, qvar_normCR_up.GetName()+"_rebinFinal", rebinFinal)
                qvar_normCR_dw = qvar_normCR_dw.Rebin(len(rebinFinal)-1, qvar_normCR_dw.GetName()+"_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(qvar_normCR_up, "qcd_hh_QCDNormCRup")
                outfileStat.WriteTObject(qvar_normCR_dw, "qcd_hh_QCDNormCRdown")

            qvar_normCR_up.SetDirectory(0)
            qvar_normCR_dw.SetDirectory(0)
            output_Dict[r]["qcd"]["QCDNormCRup"] = qvar_normCR_up
            output_Dict[r]["qcd"]["QCDNormCRdown"] = qvar_normCR_dw


        #####################################################################################################################
        ### top shape systematics in 4b region, if using 3b shape ###########################################################
        #####################################################################################################################
        if r == "44" and nbtag_top_shape_SRPred_for4b == "33" and MassRegionName == "SR"  and isMhhDistribution:   # qi
            ## ttbarShapeSRSyst_Dict = SystTools.ttbarShapeSysSR(topfileName,
            ##                                                     distributionName,
            ##                                                     signal_region = "22",
            ##                                                     compare_region = "33",
            ##                                                     btag_WP     = btag_WP,
            ##                                                     makePlots = True,
            ##                                                     verbose = False,
            ##                                                     outfileNameBase="TopShapeSRSysfit.root")
            ttbarShapeSRSyst_Dict = SystToolsSmooth.ttbarShapeSysSR(topfileName,
                                                                distributionName,
                                                                signal_region = "33",
                                                                compare_region = "22",
                                                                btag_WP     = btag_WP,
                                                                smoothing_func = top_smoothing_func,
                                                                SmoothRange = topSmoothRange,# (100, 2500),
                                                                makePlots = True,
                                                                verbose = False,
                                                                outfileNameBase="TopShapeSRSysfitSmooth.root")

            tvar_shape_up = top_r.Clone("tvar_ttbarShapeSR_up")
            #tvar_shape_up.Multiply( ttbarShapeSRSyst_Dict["fup"] )

            tvar_shape_dw = top_r.Clone("tvar_ttbarShapeSR_dw")
            #tvar_shape_dw.Multiply( ttbarShapeSRSyst_Dict["fdw"] )

            ClearNegBin(tvar_shape_up)
            ClearNegBin(tvar_shape_dw)
                    
            tvar_shape_up.Scale( top_r.Integral() / tvar_shape_up.Integral() )
            tvar_shape_dw.Scale( top_r.Integral() / tvar_shape_dw.Integral() )


            ## Now do smoothing ##########################
            if do_smoothing:
                tvar_shape_up_sm = smoothfit.smoothfit(tvar_shape_up, fitFunction = top_smoothing_func, fitRange = topSmoothRange, makePlots = False, verbose = verbose,
                                                        outfileName="top_smoothfit_"+r+"_ttbarShapeSRup.root")

                tvar_shape_dw_sm = smoothfit.smoothfit(tvar_shape_dw, fitFunction = top_smoothing_func, fitRange = topSmoothRange, makePlots = False, verbose = verbose,
                                                        outfileName="top_smoothfit_"+r+"_ttbarShapeSRedown.root")

                if addSmoothErrorBin:
                    tvar_shape_up_final = smoothfit.MakeSmoothHistoWithError(tvar_shape_up, tvar_shape_up_sm)
                    tvar_shape_dw_final = smoothfit.MakeSmoothHistoWithError(tvar_shape_dw, tvar_shape_dw_sm)
                else:
                    tvar_shape_up_final = smoothfit.MakeSmoothHisto(tvar_shape_up, tvar_shape_up_sm["nom"])
                    tvar_shape_dw_final = smoothfit.MakeSmoothHisto(tvar_shape_dw, tvar_shape_dw_sm["nom"])

                tvar_shape_up_final.Multiply( ttbarShapeSRSyst_Dict["Shape"] )
                tvar_shape_dw_final.Divide( ttbarShapeSRSyst_Dict["Shape"] )

                tvar_shape_up_final.SetNameTitle("ttbar_hh_"+r+"_ttbarShapeSRup__clone",     "ttbar_hh_"+r+"_ttbarShapeSRup__clone")
                tvar_shape_dw_final.SetNameTitle("ttbar_hh_"+r+"_ttbarShapeSRdown__clone",   "ttbar_hh_"+r+"_ttbarShapeSRdown__clone")


            else:
                tvar_shape_up_final = tvar_shape_up.Clone("ttbar_hh_"+r+"_ttbarShapeSRup__clone")
                tvar_shape_dw_final = tvar_shape_dw.Clone("ttbar_hh_"+r+"_ttbarShapeSRdown__clone")


            if rebinFinal is not None:
                tvar_shape_up_final = tvar_shape_up_final.Rebin(len(rebinFinal)-1, tvar_shape_up_final.GetName()+"_rebinFinal", rebinFinal)
                tvar_shape_dw_final = tvar_shape_dw_final.Rebin(len(rebinFinal)-1, tvar_shape_dw_final.GetName()+"_rebinFinal", rebinFinal)

            if makeOutputFiles:
                outfileStat.WriteTObject(tvar_shape_up_final, "ttbar_hh_ttbarShapeSRup")
                outfileStat.WriteTObject(tvar_shape_dw_final, "ttbar_hh_ttbarShapeSRdown")

            tvar_shape_up_final.SetDirectory(0)
            tvar_shape_dw_final.SetDirectory(0)
            output_Dict[r]["ttbar"]["ttbarShapeSRup"] = tvar_shape_up_final
            output_Dict[r]["ttbar"]["ttbarShapeSRdown"] = tvar_shape_dw_final
        
        ### close outfiles, if used ###
        if makeOutputFiles:
            outfileStat.Close()

    ### Print tables ###
    PrintTable( Nbkg_dict, Nbkg_SysList, regions)
    #print vartxt

    output_Dict['regions'] = regions
    #print output_Dict
    return 
def smoothFuncCompare(histo, fitFunction = "ExpModGauss", fitRange = (100, 3000), funcCompareRange=(900,3000), makePlots = False, plotExtra = True, verbose = False, outfileName="ExpModGaussSmoothFuncCompare.root"):


    
    colorlist = [R.kBlue, R.kGreen, R.kOrange, R.kMagenta, R.kCyan, R.kPink, (R.kAzure+1), R.kGreen+2]        

    namestr = outfileName.split(".root")[0]

    h_clone = histo.Clone()
    h_clone.SetDirectory(0)


    nominal_result = smoothfit.smoothfit(h_clone, fitFunction = fitFunction, fitRange = fitRange, makePlots = False, verbose = verbose, outfileName = fitFunction+"_"+outfileName)

    nominal_hist = smoothfit.MakeSmoothHisto(h_clone, nominal_result["nom"])
    
    results = {}
    results_hist = {}
    results_hist_ud = {}

    for theFunc in ["Exp","MJ2","MJ3","MJ4","MJ5","MJ6","MJ7","MJ8"]:
        results[theFunc] = smoothfit.smoothfit(h_clone, fitFunction = theFunc, fitRange = funcCompareRange, makePlots = False, verbose = verbose, outfileName = theFunc+"_"+outfileName)
        results_hist[theFunc] = smoothfit.MakeSmoothHisto(h_clone, results[theFunc]["nom"])

        results_hist_ud[theFunc] = {}
        for ivar in range(len(results[theFunc]["vars"])):
            results_hist_ud[theFunc]["up"+str(ivar)] = smoothfit.MakeSmoothHisto(h_clone, results[theFunc]["vars"][ivar][0])
            results_hist_ud[theFunc]["dw"+str(ivar)] = smoothfit.MakeSmoothHisto(h_clone, results[theFunc]["vars"][ivar][1])



    histo_up = nominal_hist.Clone(histo.GetName() + "_" + namestr + "_up")
    histo_up.SetDirectory(0)
    histo_dw = nominal_hist.Clone(histo.GetName() + "_" + namestr + "_dw")
    histo_dw.SetDirectory(0)

    histo_up_super = nominal_hist.Clone(histo.GetName() + "_" + namestr + "_up_super")
    histo_up_super.SetDirectory(0)
    histo_dw_super = nominal_hist.Clone(histo.GetName() + "_" + namestr + "_dw_super")
    histo_dw_super.SetDirectory(0)

    for ibin in range(1, histo.GetNbinsX()+1):
        if histo.GetBinLowEdge(ibin) + histo.GetBinWidth(ibin) < funcCompareRange[0]:
            continue
        
        deltas = []
        deltas_super = []
        for theFunc in ["Exp","MJ2","MJ3","MJ4","MJ5","MJ6","MJ7","MJ8"]:
            deltas.append( np.abs( nominal_hist.GetBinContent(ibin) - results_hist[theFunc].GetBinContent(ibin) ) )

            for ivarh in results_hist_ud[theFunc]:
                deltas_super.append( np.abs( nominal_hist.GetBinContent(ibin) - results_hist_ud[theFunc][ivarh].GetBinContent(ibin) ) )
            

        theDelta = np.max( deltas )
        theDelta_super = np.max( deltas_super )
        histo_up.SetBinContent(ibin, histo_up.GetBinContent(ibin) + theDelta)
        histo_dw.SetBinContent(ibin, histo_dw.GetBinContent(ibin) - theDelta)
        
        histo_up_super.SetBinContent(ibin, histo_up.GetBinContent(ibin) + theDelta_super)
        histo_dw_super.SetBinContent(ibin, histo_dw.GetBinContent(ibin) - theDelta_super)

    smoothFuncCompSyst = {"up":histo_up, "dw":histo_dw, "up_super":histo_up_super, "dw_super":histo_dw_super}
        
        

    if makePlots:
        f = R.TFile(outfileName, "RECREATE")
        
        c=R.TCanvas("c1","c1")
        #R.SetOwnership(c,False)
        leg = R.TLegend(0.1,0.7,0.48,0.9)
        leg.SetFillColor(0)
    
        h_clone.SetLineColor(R.kBlack)
        h_clone.Draw()
        leg.AddEntry(histo, "Histogram", "L")

        icol = 0
        ivar0 = True
        err_hist_ratio = None


        err_hist = smoothfit.MakeSmoothHisto(histo, nominal_result["nom"])
        err_hist.SetDirectory(0)

        for ivar in range(len(nominal_result["vars"])):
            err_hist_up = smoothfit.MakeSmoothHisto(histo, nominal_result["vars"][ivar][0])
            err_hist_dw = smoothfit.MakeSmoothHisto(histo, nominal_result["vars"][ivar][1])

            for ibin in range(1, err_hist.GetNbinsX()+1):
                err_val = np.max( np.abs( [ err_hist.GetBinContent(ibin) - err_hist_up.GetBinContent(ibin), err_hist.GetBinContent(ibin) - err_hist_dw.GetBinContent(ibin)] ) )
                err_hist.SetBinError(ibin, np.sqrt( err_hist.GetBinError(ibin)**2 + err_val**2) )

        err_hist_ratio = err_hist.Clone("err_hist_ratio__"+namestr)
        err_hist_ratio.SetDirectory(0)
        err_hist_ratio.Divide( nominal_result["nom"] )


        err_hist.SetFillColor(R.kBlack)
        err_hist.SetFillStyle(3001)
        err_hist.Draw("sameE3")
        leg.AddEntry(err_hist, "smoothing error", "F")

        
        for theFunc in ["Exp","MJ2","MJ3","MJ4","MJ5","MJ6","MJ7","MJ8"]:

            #print results[theFunc]["nom"], results[theFunc]["nom"].Eval(1000), results[theFunc]["nom"].Eval(2000), results[theFunc]["nom"].Eval(3000)

            results[theFunc]["nom"].SetLineColor( colorlist[icol] )
            results[theFunc]["nom"].Draw("same")
            leg.AddEntry(results[theFunc]["nom"], theFunc, "L")


            if plotExtra:
                for ivar in range(len(results[theFunc]["vars"])):
                    results[theFunc]["vars"][ivar][0].SetLineColor( R.kGray+2 )
                    results[theFunc]["vars"][ivar][0].Draw("same")
                    results[theFunc]["vars"][ivar][1].SetLineColor( R.kGray+2 )
                    results[theFunc]["vars"][ivar][1].Draw("same")
                    
            

            icol += 1
            
        if plotExtra:
            results_hist_ud["MJ2"]["up0"].SetLineColor( R.kGray+2 )
            leg.AddEntry(results_hist_ud["MJ2"]["up0"], "Param Variations", "L")
        leg.Draw()
        


        c2=R.TCanvas("c2","c2")
        #R.SetOwnership(c,False)
        print "err_hist_ratio",err_hist_ratio
        err_hist_ratio.SetFillColor(R.kBlack)
        err_hist_ratio.SetFillStyle(3001)
        err_hist_ratio.Draw("E2")

        icol = 0
        f_ratio = {}


        delta_ratio_super = {}
        for theFunc in ["Exp","MJ2","MJ3","MJ4","MJ5","MJ6","MJ7","MJ8"]:

            h_ratio  = results[theFunc]["nom"].GetHistogram()
            h_ratio.Divide( nominal_result["nom"] )
            h_ratio.SetDirectory(0)

            h_ratio.SetLineColor( colorlist[icol] )
            h_ratio.Draw("same")

            if plotExtra:
                for ivar in range(len(results[theFunc]["vars"])):
                    h_ratio_ud = results[theFunc]["vars"][ivar][0].GetHistogram()
                    h_ratio_ud.Divide( nominal_result["nom"] )
                    h_ratio_ud.SetDirectory(0)
                    h_ratio_ud.SetLineColor(R.kGray+2)
                    h_ratio_ud.Draw("same")

                    delta_ratio_super[theFunc+"_"+str(ivar)+"_up"] = h_ratio_ud.GetBinContent( h_ratio_ud.FindBin(3000) )


                    h_ratio_ud = results[theFunc]["vars"][ivar][1].GetHistogram()
                    h_ratio_ud.Divide( nominal_result["nom"] )
                    h_ratio_ud.SetDirectory(0)
                    h_ratio_ud.SetLineColor(R.kGray+2)
                    h_ratio_ud.Draw("same")

                    delta_ratio_super[theFunc+"_"+str(ivar)+"_dw"] = h_ratio_ud.GetBinContent( h_ratio_ud.FindBin(3000) )
            
            #print f_copy, f_ratio[theFunc], f_ratio[theFunc].Eval(1000), f_ratio[theFunc].Eval(2000), f_ratio[theFunc].Eval(3000)
            icol += 1

        leg.Draw()
        #for drs in delta_ratio_super:
        #    print drs, delta_ratio_super[drs]

        f.WriteTObject(c)
        f.WriteTObject(c2)
        f.Close()
        
    return smoothFuncCompSyst
예제 #8
0
def HistoAnalysis(datafileName="hist_data.root",
                  topfileName="hist_ttbar.root",
                  distributionName="DiJetMass",
                  n_trkjet=["4", "4"],
                  n_btag=["4", "3"],
                  btag_WP="77",
                  NRebin=1,
                  use_one_top_nuis=False,
                  use_scale_top_2b=False,
                  nbtag_top_shape_normFit=None,
                  nbtag_top_shape_SRPred=None,
                  rebinFinal=None,
                  verbose=False):

    global func1
    global func2

    ##### Parse Inputs ############################################
    dist_name = distributionName

    num_trkjet = np.asarray(n_trkjet)
    if num_trkjet.shape == ():
        num_trkjet = np.asarray([n_trkjet])

    num_btag = np.asarray(n_btag)
    if num_btag.shape == ():
        num_btag = np.asarray([n_btag])
    if num_btag.shape != num_trkjet.shape:
        print "Must have same number of track jet and b-tag regions specified"
        sys.exit(0)

    btag_WP = btag_WP

    n_rebin = NRebin

    nbtag_top_shape = nbtag_top_shape_SRPred
    topShape_nbtag = nbtag_top_shape
    if nbtag_top_shape == None:
        topShape_nbtag = num_btag

    useOneTopNuis = use_one_top_nuis

    scaleTop2b = use_scale_top_2b

    n_channels = num_trkjet.shape[0]

    regions = [num_trkjet[i] + num_btag[i] for i in range(n_channels)]
    ##################################################################

    ##### Do Background Fits ############################################
    bkgFitResults = BkgFit.BackgroundFit(
        datafileName=datafileName,
        topfileName=topfileName,
        distributionName="LeadCaloJetM",
        n_trkjet=n_trkjet,
        n_btag=n_btag,
        btag_WP=btag_WP,
        NRebin=NRebin,
        use_one_top_nuis=use_one_top_nuis,
        use_scale_top_2b=use_scale_top_2b,
        nbtag_top_shape=nbtag_top_shape_normFit,
        makePlots=True,
        verbose=verbose)

    pvars = bkgFitResults["pvars"]
    ##################################################################

    ##### Get Signal Region Histograms ################################
    datafile = R.TFile(datafileName, "READ")
    topfile = R.TFile(topfileName, "READ")

    histos = {}

    # collect all histograms
    for r in ["44", "43", "42", "33", "32"]:
        folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP,
                              "SR")  #folder( r[0], r[1], btag_WP)

        data_r = datafile.Get(folder_r).Clone("data_" + r)
        data_r.SetDirectory(0)

        top_r = topfile.Get(folder_r).Clone("top_" + r)
        top_r.SetDirectory(0)

        for ibin in range(1, top_r.GetNbinsX() + 1):
            if top_r.GetBinContent(ibin) < 0:
                top_r.SetBinContent(ibin, 0)
                top_r.SetBinError(ibin, 0)

        histos[r] = {"data": data_r, "top": top_r}

    datafile.Close()
    topfile.Close()
    ##################################################################

    ##### scaling and subtractions #################################
    for ir in range(len(regions)):
        r = regions[ir]

        outfileStat = R.TFile("outfile_boosted_" + r + ".root", "RECREATE")

        r_2b = r[0] + "2"
        r_3b = r[0] + "3"

        top_2b = histos[r_2b]["top"].Clone("top_2b__" + r)
        if scaleTop2b:
            top_2b.Scale((bkgFitResults["topscale"][0] if use_one_top_nuis else
                          bkgFitResults["topscale"][ir]))

        qcd_r = histos[r_2b]["data"].Clone("qcd__" + r)
        qcd_int = qcd_r.Integral()
        qcd_r.Add(
            top_2b, -1
        )  # added by Qi --- we still want top to be subtracted, given that their fraction is increasing in Run 2.

        top_r = histos[r]["top"].Clone("top__" + r)
        if (nbtag_top_shape == "3") and (
                r == "44"
        ):  # the 3b top shape is only used during the SR prediction for 44 region
            temp_scaler = top_r.Integral() / histos[r_3b]["top"].Integral()
            top_r = histos[r_3b]["top"].Clone("top__" + r)
            top_r.Scale(temp_scaler)
        top_int = top_r.Integral()

        mu_qcd = bkgFitResults["muqcd"][ir]
        top_scale = (bkgFitResults["topscale"][0]
                     if use_one_top_nuis else bkgFitResults["topscale"][ir])

        qcd_r.Scale(mu_qcd)
        top_r.Scale(top_scale)

        ## Now do smoothing

        qcd_sm = smoothfit.smoothfit(qcd_r,
                                     fitFunction="Exp",
                                     fitRange=(900, 2000),
                                     makePlots=True,
                                     verbose=verbose,
                                     outfileName="qcd_smoothfit_" + r +
                                     ".root")
        top_sm = smoothfit.smoothfit(top_r,
                                     fitFunction="Exp",
                                     fitRange=(850, 1200),
                                     makePlots=True,
                                     verbose=verbose,
                                     outfileName="top_smoothfit_" + r +
                                     ".root")

        qcd_final = smoothfit.MakeSmoothHisto(qcd_r, qcd_sm["nom"])
        top_final = smoothfit.MakeSmoothHisto(top_r, top_sm["nom"])

        if rebinFinal is not None:
            qcd_final = qcd_final.Rebin(
                len(rebinFinal) - 1,
                qcd_final.GetName() + "_rebinFinal", rebinFinal)
            top_final = top_final.Rebin(
                len(rebinFinal) - 1,
                top_final.GetName() + "_rebinFinal", rebinFinal)

        # outfileStat.WriteTObject(qcd_final, "qcd_hh_nominal","Overwrite")
        # outfileStat.WriteTObject(top_final, "top_hh_nominal","Overwrite")

        outfileStat.WriteTObject(qcd_final, "qcd_hh", "Overwrite")
        outfileStat.WriteTObject(top_final, "ttbar_hh", "Overwrite")

        ### propagate correlated systematics from the smoothing procedure---> these "replace" the stat error on the bins  #############
        for ivar in range(len(qcd_sm["vars"])):
            qup = qcd_sm["vars"][ivar][0]
            qdw = qcd_sm["vars"][ivar][1]

            qcd_r_qup = smoothfit.MakeSmoothHisto(qcd_r, qup)
            qcd_r_qdw = smoothfit.MakeSmoothHisto(qcd_r, qdw)

            if rebinFinal is not None:
                qcd_r_qup = qcd_r_qup.Rebin(
                    len(rebinFinal) - 1,
                    qcd_r_qup.GetName() + "_rebinFinal", rebinFinal)
                qcd_r_qdw = qcd_r_qdw.Rebin(
                    len(rebinFinal) - 1,
                    qcd_r_qdw.GetName() + "_rebinFinal", rebinFinal)

            outfileStat.WriteTObject(qcd_r_qup,
                                     "qcd_hh_smoothQ" + str(ivar) + "Up",
                                     "Overwrite")
            outfileStat.WriteTObject(qcd_r_qdw,
                                     "qcd_hh_smoothQ" + str(ivar) + "Down",
                                     "Overwrite")

        for ivar in range(len(top_sm["vars"])):
            tup = top_sm["vars"][ivar][0]
            tdw = top_sm["vars"][ivar][1]

            top_r_tup = smoothfit.MakeSmoothHisto(top_r, tup)
            top_r_tdw = smoothfit.MakeSmoothHisto(top_r, tdw)

            if rebinFinal is not None:
                top_r_tup = top_r_tup.Rebin(
                    len(rebinFinal) - 1,
                    top_r_tup.GetName() + "_rebinFinal", rebinFinal)
                top_r_tdw = top_r_tdw.Rebin(
                    len(rebinFinal) - 1,
                    top_r_tdw.GetName() + "_rebinFinal", rebinFinal)

            # outfileStat.WriteTObject(top_r_tup, "top_hh_smoothT"+str(ivar)+"Up","Overwrite")
            # outfileStat.WriteTObject(top_r_tdw, "top_hh_smoothT"+str(ivar)+"Down","Overwrite")

            outfileStat.WriteTObject(top_r_tup,
                                     "ttbar_hh_smoothT" + str(ivar) + "Up",
                                     "Overwrite")
            outfileStat.WriteTObject(top_r_tdw,
                                     "ttbar_hh_smoothT" + str(ivar) + "Down",
                                     "Overwrite")

        ### propagate correlated systematics from normalization fits for mu_qcd and top_scale ###############
        for ivar in range(len(pvars)):
            for iUD in range(2):
                mu_qcd_var = pvars[ivar][iUD][ir]
                top_scale_var = pvars[ivar][iUD][
                    n_channels + (0 if use_one_top_nuis else ir)]

                qvar = qcd_r.Clone("qvar")
                qvar.Scale(mu_qcd_var * qcd_int / qvar.Integral())

                tvar = top_r.Clone("tvar")
                tvar.Scale(top_scale_var * top_int / tvar.Integral())

                ## Now do smoothing

                qvar_sm = smoothfit.smoothfit(qvar,
                                              fitFunction="Exp",
                                              fitRange=(900, 2000),
                                              makePlots=False,
                                              verbose=verbose,
                                              outfileName="qcd_smoothfit_" +
                                              r + "_Norm" + str(ivar) +
                                              str(iUD) + ".root")
                tvar_sm = smoothfit.smoothfit(tvar,
                                              fitFunction="Exp",
                                              fitRange=(850, 1200),
                                              makePlots=False,
                                              verbose=verbose,
                                              outfileName="top_smoothfit_" +
                                              r + "_Norm" + str(ivar) +
                                              str(iUD) + ".root")

                qvar_final = smoothfit.MakeSmoothHisto(qvar, qvar_sm["nom"])
                tvar_final = smoothfit.MakeSmoothHisto(tvar, tvar_sm["nom"])

                if rebinFinal is not None:
                    qvar_final = qvar_final.Rebin(
                        len(rebinFinal) - 1,
                        qvar_final.GetName() + "_rebinFinal", rebinFinal)
                    tvar_final = tvar_final.Rebin(
                        len(rebinFinal) - 1,
                        tvar_final.GetName() + "_rebinFinal", rebinFinal)

                UpDw = ("Up" if iUD == 0 else "Down")
                outfileStat.WriteTObject(qvar_final,
                                         "qcd_hh_normY" + str(ivar) + UpDw,
                                         "Overwrite")
                # outfileStat.WriteTObject(tvar_final, "top_hh_normY"+str(ivar)+UpDw,"Overwrite")
                outfileStat.WriteTObject(tvar_final,
                                         "ttbar_hh_normY" + str(ivar) + UpDw,
                                         "Overwrite")

        outfileStat.Close()

        ## if False:
        ##     pred_final = qcd_final.Clone("pred_final__"+r)
        ##     pred_final.Add( top_final )

        ##     func1 = qcd_sm["nom"]
        ##     func2 = top_sm["nom"]

        ##     pred_sm = R.TF1("pred_sm", FuncSum, 900, 3000)

        ##     pred_sm.Draw("same")
        ##     top_sm["nom"].Draw("same")

        ##     pred_final_raw = qcd_r.Clone("qcd_final_raw__"+r)
        ##     pred_final_raw.Add(top_r)

        ##     outfile = R.TFile("outfile_"+r+".root","RECREATE")

        ##     c=R.TCanvas()
        ##     pred_final_raw.Draw("HIST")
        ##     top_r.SetLineColor(R.kBlack)
        ##     top_r.SetFillColor(R.kGreen)
        ##     top_r.Draw("sameHIST")

        ##     pred_sm.Draw("same")
        ##     top_sm["nom"].Draw("same")

        ##     c.Write()

        ##     c=R.TCanvas()

        ##     pred_final.Draw("HIST")

        ##     top_final.SetLineColor(R.kBlack)
        ##     top_final.SetFillColor(R.kGreen)

        ##     top_final.Draw("sameHIST")

        ##     c.Write()

        ##     outfile.Close()

    return
예제 #9
0
def ttbarShapeSysSR(
        topfileName="hist_ttbar.root",
        distributionName="mHH_l",
        signal_region="33",
        compare_region="44",
        btag_WP="77",
        smoothing_func="Exp",
        SmoothRange=(1200, 3000),  # (100, 2500),
        makePlots=False,
        verbose=False,
        outfileNameBase="TopShapeSRSysfitSmooth.root"):

    global rfunc1
    global rfunc2

    topfile = R.TFile(topfileName, "READ")

    ttbarShapeSRSyst_Dict = {}

    colorlist = [
        R.kGreen, R.kOrange, R.kMagenta, R.kCyan, R.kPink, (R.kAzure + 1),
        R.kGreen + 2, R.kOrange + 5
    ]

    ## get top SR shape
    folder_sig = HistLocStr(distributionName, signal_region[0],
                            signal_region[1], btag_WP,
                            "SR")  #folder( r[0], r[1], btag_WP)
    top_sig = topfile.Get(folder_sig).Clone("top_sig_" + signal_region)
    top_sig.SetDirectory(0)
    top_sig.Rebin(5)

    ## get top comparison shape
    folder_comp = HistLocStr(distributionName, compare_region[0],
                             compare_region[1], btag_WP,
                             "SR")  #folder( r[0], r[1], btag_WP)
    top_comp = topfile.Get(folder_comp).Clone("top_comp_" + compare_region)
    top_comp.SetDirectory(0)
    top_comp.Rebin(5)

    ## remove negative values
    ## assume same binning, else division won't work later
    for ibin in range(1, top_sig.GetNbinsX() + 1):
        if top_sig.GetBinContent(ibin) < 0:
            top_sig.SetBinContent(ibin, 0)
            top_sig.SetBinError(ibin, 0)

        if top_comp.GetBinContent(ibin) < 0:
            top_comp.SetBinContent(ibin, 0)
            top_comp.SetBinError(ibin, 0)

    ## normalize to same area
    top_sig.Scale(top_comp.Integral() / top_sig.Integral())

    c = R.TCanvas("c1_topsys", "c1_topsys")
    xleg, yleg = 0.52, 0.7
    leg = R.TLegend(xleg, yleg, xleg + 0.3, yleg + 0.2)
    leg.SetFillColor(0)
    leg.SetBorderSize(0)
    leg.SetMargin(0.3)
    top_comp.SetXTitle("m_{JJ} [GeV]")
    top_comp.SetYTitle("Entries")
    top_comp.Draw("E1")
    leg.AddEntry(top_comp, "Top Comparison Distribution", "LP")

    #################################
    ## smooth bkg and data
    ##################################
    top_comp_sm = smoothfit.smoothfit(
        top_comp,
        fitFunction=smoothing_func,
        fitRange=SmoothRange,
        makePlots=False,
        verbose=False,
        outfileName="top_comp_smoothfit_TopShape4b.root")
    top_comp_sm_h = smoothfit.MakeSmoothHisto(top_comp, top_comp_sm["nom"])

    top_comp_sm["nom"].SetLineColor(R.kBlack)
    top_comp_sm["nom"].Draw("same")
    leg.AddEntry(top_comp_sm["nom"], "Top Comparison Distribution Smooth", "L")

    top_sig_sm = smoothfit.smoothfit(
        top_sig,
        fitFunction=smoothing_func,
        fitRange=SmoothRange,
        makePlots=False,
        verbose=False,
        outfileName="top_sig_smoothfit_TopShape4.root")
    top_sig_sm_h = smoothfit.MakeSmoothHisto(top_sig, top_sig_sm["nom"])

    top_sig_sm["nom"].SetLineColor(R.kBlue)
    top_sig_sm["nom"].Draw("same")
    leg.AddEntry(top_sig_sm["nom"], "Top Nominal Distribution Smooth", "L")

    rfunc1 = top_comp_sm["nom"]
    rfunc2 = top_sig_sm["nom"]
    xMax = top_comp.GetXaxis().GetBinUpEdge(top_comp.GetXaxis().GetNbins())
    ratio_sm = R.TF1("ratio_topsys_sm", rfunc_ratio, SmoothRange[0], xMax, 0)

    for ivar in range(len(top_comp_sm["vars"])):
        dup = top_comp_sm["vars"][ivar][0]
        ddw = top_comp_sm["vars"][ivar][1]

        dup.SetLineColor(colorlist[ivar])
        ddw.SetLineColor(colorlist[ivar])

        dup.Draw("same")
        ddw.Draw("same")
        leg.AddEntry(dup, "Top Comparison Smooth Variation", "L")

        top_comp_r_qup = smoothfit.MakeSmoothHisto(top_comp, dup)
        top_comp_r_qdw = smoothfit.MakeSmoothHisto(top_comp, ddw)

        for ibin in range(1, top_comp_sm_h.GetNbinsX() + 1):
            err_val = np.max(
                np.abs([
                    top_comp_sm_h.GetBinContent(ibin) -
                    top_comp_r_qup.GetBinContent(ibin),
                    top_comp_sm_h.GetBinContent(ibin) -
                    top_comp_r_qdw.GetBinContent(ibin)
                ]))
            top_comp_sm_h.SetBinError(
                ibin, np.sqrt(top_comp_sm_h.GetBinError(ibin)**2 + err_val**2))

    c.SetLogy(1)
    leg.Draw("same")
    c.SaveAs(
        outfileNameBase.split(".root")[0] + "_sig" + signal_region + "_comp" +
        compare_region + ".root")
    c.SaveAs(
        outfileNameBase.split(".root")[0] + "_sig" + signal_region + "_comp" +
        compare_region + ".pdf")
    c.Close()

    h_ratio_cr_nom = top_comp_sm_h.Clone("top_comp_sm_h_TopShape4b")
    h_ratio_cr_nom.Divide(top_comp_sm["nom"])
    h_ratio_cr_nom.SetDirectory(0)

    h_ratio_cr = top_comp_sm["nom"].GetHistogram()
    h_ratio_cr.Divide(top_sig_sm["nom"])
    h_ratio_cr.SetDirectory(0)

    ttbarShapeSRSyst_Dict["Shape"] = ratio_sm

    c2 = R.TCanvas("c2_topsys", "c2_topsys")
    leg = R.TLegend(0.2, 0.7, 0.5, 0.9)
    leg.SetFillColor(0)
    h_ratio_cr_nom.SetFillColor(R.kBlack)
    h_ratio_cr_nom.SetFillStyle(3004)
    h_ratio_cr_nom.SetMarkerSize(0)
    h_ratio_cr_nom.GetXaxis().SetRangeUser(1000, 3000)
    h_ratio_cr_nom.GetYaxis().SetRangeUser(0, 3)
    h_ratio_cr_nom.GetXaxis().SetLabelSize(0.04)
    h_ratio_cr_nom.GetYaxis().SetLabelSize(0.04)
    h_ratio_cr_nom.SetXTitle("m_{JJ} [GeV]")
    h_ratio_cr_nom.SetYTitle("Ratio")
    h_ratio_cr_nom.Draw("E2")
    leg.AddEntry(h_ratio_cr_nom, "Nominal", "LF")

    h_ratio_cr.SetLineColor(R.kBlue)
    h_ratio_cr.Draw("same")
    leg.AddEntry(h_ratio_cr, "Predicted", "L")

    #ratio_sm.Draw("same")

    leg.Draw("same")
    c2.SaveAs(
        outfileNameBase.split(".root")[0] + "_sig" + signal_region + "_comp" +
        compare_region + "_ratio.root")
    c2.SaveAs(
        outfileNameBase.split(".root")[0] + "_sig" + signal_region + "_comp" +
        compare_region + "_ratio.pdf")
    c2.Close()

    topfile.Close()

    return ttbarShapeSRSyst_Dict
예제 #10
0
def QCDSystematics(
        datafileName="hist_data.root",
        topfileName="hist_ttbar.root",
        zjetfileName="hist_Zjets.root",
        distributionName="mHH_l",
        n_trkjet=["4", "3", "2"],
        n_btag=["4", "3", "2"],
        btag_WP="77",
        mu_qcd_vals=[1.0, 1.0],
        topscale_vals=[1.0, 1.0],
        NRebin=1,
        smoothing_func="Dijet",
        SmoothRange=(1100, 3000),  # (100, 2500),
        use_one_top_nuis=False,
        use_scale_top_0b=False,
        nbtag_top_shape_for4b=None,
        makePlots=False,
        verbose=False,
        outfileNameBase="QCDSysfitSmooth.root"):

    global rfunc1
    global rfunc2

    ##### Parse Inputs ############################################
    dist_name = distributionName

    num_trkjet = np.asarray(n_trkjet)
    if num_trkjet.shape == ():
        num_trkjet = np.asarray([n_trkjet])

    num_btag = np.asarray(n_btag)
    if num_btag.shape == ():
        num_btag = np.asarray([n_btag])
    if num_btag.shape != num_trkjet.shape:
        print "Must have same number of track jet and b-tag regions specified"
        sys.exit(0)

    btag_WP = btag_WP

    n_rebin = NRebin

    useOneTopNuis = use_one_top_nuis

    scaleTop0b = use_scale_top_0b

    n_channels = num_trkjet.shape[0]

    regions = [num_trkjet[i] + num_btag[i] for i in range(n_channels)]
    ##################################################################

    colorlist = [
        R.kGreen, R.kOrange, R.kMagenta, R.kCyan, R.kPink, (R.kAzure + 1),
        R.kGreen + 2, R.kOrange + 5
    ]

    ##### Get Signal Region Histograms ################################
    datafile = R.TFile(datafileName, "READ")
    topfile = R.TFile(topfileName, "READ")
    zjetfile = (R.TFile(zjetfileName, "READ")
                if zjetfileName != None else None)

    histos = {}

    # collect all histograms
    for r in ["44", "33", "22", "40", "30", "20"]:
        folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP,
                              "CR")  #folder( r[0], r[1], btag_WP)

        data_r = datafile.Get(folder_r).Clone("data_" + r)
        data_r.SetDirectory(0)

        top_r = topfile.Get(folder_r).Clone("top_" + r)
        top_r.SetDirectory(0)

        zjet_r = CheckAndGet(zjetfile, folder_r, top_r).Clone("zjet_" + r)
        zjet_r.SetDirectory(0)

        for ibin in range(1, top_r.GetNbinsX() + 1):
            if top_r.GetBinContent(ibin) < 0:
                top_r.SetBinContent(ibin, 0)
                top_r.SetBinError(ibin, 0)

        data_r.Rebin(n_rebin)
        top_r.Rebin(n_rebin)
        zjet_r.Rebin(n_rebin)

        histos[r] = {"data": data_r, "top": top_r, "zjet": zjet_r}

    datafile.Close()
    topfile.Close()
    if zjetfile != None:
        zjetfile.Close()
    ##################################################################

    ####### outpue object ###################
    QCDSyst_Dict = {}

    ##### scaling and subtractions #################################
    for ir in range(len(regions)):
        r = regions[ir]

        r_0b = r[0] + "0"

        top_0b = histos[r_0b]["top"].Clone("top_0b__" + r)
        if scaleTop0b:
            top_0b.Scale(
                (topscale_vals[0] if use_one_top_nuis else topscale_vals[ir]))

        zjet_0b = histos[r_0b]["zjet"].Clone("zjet_0b__" + r)

        qcd_r = histos[r_0b]["data"].Clone("qcd__" + r)
        qcd_r.Add(
            top_0b, -1
        )  # added by Qi --- we still want top to be subtracted, given that their fraction is increasing in Run 2.
        qcd_r.Add(zjet_0b, -1)
        qcd_int = qcd_r.Integral()

        if nbtag_top_shape_for4b != None:
            top_r = histos[nbtag_top_shape_for4b]["top"].Clone("top__" + r)
            top_r.Scale(histos[r]["top"].Integral() /
                        top_r.Integral())  #scale to correct norm for region
        else:
            top_r = histos[r]["top"].Clone("top__" + r)

        top_int = top_r.Integral()

        zjet_r = histos[r]["zjet"].Clone("zjet__" + r)

        mu_qcd = mu_qcd_vals[ir]
        top_scale = (topscale_vals[0]
                     if use_one_top_nuis else topscale_vals[ir])

        qcd_r.Scale(mu_qcd)
        top_r.Scale(top_scale)

        N_qcd_r = qcd_r.Integral()

        #now do ratio
        bkg_r = qcd_r.Clone("bkg__" + r)
        bkg_r.Add(top_r)
        bkg_r.Add(zjet_r)

        N_bkg_r = bkg_r.Integral()

        Err_N_data_CR_r = R.Double(0)
        N_data_CR_r = histos[r]["data"].IntegralAndError(
            0, histos[r]["data"].GetNbinsX() + 1, Err_N_data_CR_r)

        bkg_r.Scale(histos[r]["data"].Integral() / bkg_r.Integral())

        c = R.TCanvas("c1_cr_" + r, "c1_cr_" + r)
        xleg, yleg = 0.52, 0.7
        leg = R.TLegend(xleg, yleg, xleg + 0.3, yleg + 0.2)
        leg.SetFillColor(0)
        leg.SetBorderSize(0)
        leg.SetMargin(0.3)
        histos[r]["data"].SetXTitle("m_{JJ} [GeV]")
        histos[r]["data"].SetYTitle("Entries")
        histos[r]["data"].Draw("E1")
        leg.AddEntry(histos[r]["data"], "CR data", "LP")

        ##################################
        ## smooth bkg and data
        ##################################
        data_sm = smoothfit.smoothfit(histos[r]["data"],
                                      fitFunction=smoothing_func,
                                      fitRange=SmoothRange,
                                      makePlots=False,
                                      verbose=False,
                                      useLikelihood=True,
                                      outfileName="data_smoothfit_CRsyst_" +
                                      r + ".root")
        data_sm_h = smoothfit.MakeSmoothHisto(histos[r]["data"],
                                              data_sm["nom"])

        data_sm["nom"].SetNameTitle("data_smoothfit_CRsyst_" + r,
                                    "data_smoothfit_CRsyst_" + r)
        data_sm["nom"].SetLineColor(R.kBlack)
        data_sm["nom"].Draw("same")
        leg.AddEntry(data_sm["nom"], "CR data smoothed", "L")

        bkg_sm = smoothfit.smoothfit(bkg_r,
                                     fitFunction=smoothing_func,
                                     fitRange=SmoothRange,
                                     makePlots=False,
                                     verbose=False,
                                     outfileName="bkg_smoothfit_CRsyst_" + r +
                                     ".root")
        bkg_sm_h = smoothfit.MakeSmoothHisto(bkg_r, bkg_sm["nom"])

        bkg_sm["nom"].SetNameTitle("bkg_smoothfit_CRsyst_" + r,
                                   "bkg_smoothfit_CRsyst_" + r)
        bkg_sm["nom"].SetLineColor(R.kBlue)
        bkg_sm["nom"].Draw("same")
        leg.AddEntry(bkg_sm["nom"], "CR Prediction smoothed", "L")

        rfunc1 = data_sm["nom"]
        rfunc2 = bkg_sm["nom"]
        xMax = histos[r]["data"].GetXaxis().GetBinUpEdge(
            histos[r]["data"].GetXaxis().GetNbins())
        ratio_sm = R.TF1("ratio_crsys_sm" + r, rfunc_ratio, SmoothRange[0],
                         xMax, 0)
        ## ratio_sm.SetLineColor(R.kGray)

        for ivar in range(len(data_sm["vars"])):
            dup = data_sm["vars"][ivar][0]
            ddw = data_sm["vars"][ivar][1]

            dup.SetLineColor(colorlist[ivar])
            ddw.SetLineColor(colorlist[ivar])

            dup.Draw("same")
            ddw.Draw("same")
            leg.AddEntry(dup, "CR data smoothed variation", "L")

            data_r_qup = smoothfit.MakeSmoothHisto(histos[r]["data"], dup)
            data_r_qdw = smoothfit.MakeSmoothHisto(histos[r]["data"], ddw)

            for ibin in range(1, data_sm_h.GetNbinsX() + 1):
                err_val = np.max(
                    np.abs([
                        data_sm_h.GetBinContent(ibin) -
                        data_r_qup.GetBinContent(ibin),
                        data_sm_h.GetBinContent(ibin) -
                        data_r_qdw.GetBinContent(ibin)
                    ]))
                data_sm_h.SetBinError(
                    ibin, np.sqrt(data_sm_h.GetBinError(ibin)**2 + err_val**2))

        c.SetLogy(1)
        leg.Draw("same")
        c.SaveAs(outfileNameBase.split(".root")[0] + "_" + r + ".root")
        c.SaveAs(outfileNameBase.split(".root")[0] + "_" + r + ".pdf")
        c.Close()

        h_ratio_cr_nom = data_sm_h.Clone("data_sm_h_CRsyst_" + r)
        h_ratio_cr_nom.Divide(data_sm["nom"])
        h_ratio_cr_nom.SetDirectory(0)

        h_ratio_cr = data_sm["nom"].GetHistogram()
        h_ratio_cr.Divide(bkg_sm["nom"])
        h_ratio_cr.SetDirectory(0)

        QCDSyst_Dict["Shape_" + r] = ratio_sm

        #scale is max of ratio non-unity and CR stat error
        QCDSyst_Dict["Scale_" + r] = np.max(
            np.abs([(N_bkg_r - N_data_CR_r) / N_bkg_r,
                    (Err_N_data_CR_r / N_data_CR_r),
                    _extraNormCRSysDict.get(r, 0.)]))
        print "Scale_" + r, QCDSyst_Dict[
            "Scale_" + r], N_bkg_r, N_data_CR_r, Err_N_data_CR_r, (
                N_bkg_r - N_data_CR_r) / N_bkg_r, Err_N_data_CR_r / N_data_CR_r

        c2 = R.TCanvas("c2_cr_" + r, "c2_cr_" + r)
        leg = R.TLegend(0.2, 0.7, 0.5, 0.9)
        leg.SetFillColor(0)
        h_ratio_cr_nom.SetFillColor(R.kBlack)
        h_ratio_cr_nom.SetFillStyle(3004)
        h_ratio_cr_nom.SetMarkerSize(0)
        h_ratio_cr_nom.GetXaxis().SetRangeUser(1000, 3000)
        h_ratio_cr_nom.GetYaxis().SetRangeUser(0, 3)
        h_ratio_cr_nom.GetXaxis().SetLabelSize(0.04)
        h_ratio_cr_nom.GetYaxis().SetLabelSize(0.04)
        h_ratio_cr_nom.SetXTitle("m_{JJ} [GeV]")
        h_ratio_cr_nom.SetYTitle("Ratio")
        h_ratio_cr_nom.Draw("E2")
        leg.AddEntry(h_ratio_cr_nom, "CR data", "LF")

        h_ratio_cr.SetLineColor(R.kBlue)
        h_ratio_cr.Draw("same")
        leg.AddEntry(h_ratio_cr, "CR Prediction", "L")

        ratio_sm.Draw("same")
        leg.Draw("same")
        c2.SaveAs(outfileNameBase.split(".root")[0] + "_ratio_" + r + ".root")
        c2.SaveAs(outfileNameBase.split(".root")[0] + "_ratio_" + r + ".pdf")
        c2.Close()

    datafile.Close()
    topfile.Close()

    return QCDSyst_Dict
예제 #11
0
def HistoAnalysis(datafileName="hist_data.root",
                  topfileName="hist_ttbar.root",
                  distributionName= "DiJetMass",
                  n_trkjet  = ["4","4"],
                  n_btag    = ["4","3"],
                  btag_WP     = "77",
                  NRebin = 1,
                  use_one_top_nuis = False,
                  use_scale_top_2b = False,
                  nbtag_top_shape_normFit = None,
                  nbtag_top_shape_SRPred = None,
                  rebinFinal = None,
                  verbose = False):

    global func1
    global func2
    
    ##### Parse Inputs ############################################
    dist_name   = distributionName
    
    num_trkjet  = np.asarray(n_trkjet)
    if num_trkjet.shape==():
        num_trkjet = np.asarray([n_trkjet])

    num_btag    = np.asarray(n_btag)
    if num_btag.shape==():
        num_btag = np.asarray([n_btag])
    if num_btag.shape!=num_trkjet.shape:
        print "Must have same number of track jet and b-tag regions specified"
        sys.exit(0)
  
    btag_WP     = btag_WP
    
    n_rebin     = NRebin

    nbtag_top_shape = nbtag_top_shape_SRPred
    topShape_nbtag = nbtag_top_shape
    if nbtag_top_shape == None:
        topShape_nbtag = num_btag

    useOneTopNuis = use_one_top_nuis

    scaleTop2b = use_scale_top_2b

    n_channels = num_trkjet.shape[0]

    regions = [ num_trkjet[i]+num_btag[i] for i in range(n_channels) ]
    ##################################################################




    
    ##### Do Background Fits ############################################
    bkgFitResults = BkgFit. BackgroundFit(datafileName=datafileName,
                                      topfileName=topfileName,
                                      distributionName= "LeadCaloJetM",
                                      n_trkjet  = n_trkjet,
                                      n_btag    = n_btag,
                                      btag_WP     = btag_WP,
                                      NRebin = NRebin,
                                      use_one_top_nuis = use_one_top_nuis,
                                      use_scale_top_2b = use_scale_top_2b,
                                      nbtag_top_shape = nbtag_top_shape_normFit,
                                      makePlots = True,
                                      verbose = verbose )

    pvars = bkgFitResults["pvars"]
    ##################################################################



    

    ##### Get Signal Region Histograms ################################
    datafile = R.TFile(datafileName,"READ")
    topfile  = R.TFile(topfileName,"READ")


    histos = {}
    
    # collect all histograms
    for r in ["44","43","42","33","32"]:
        folder_r = HistLocStr(dist_name, r[0], r[1], btag_WP, "SR")  #folder( r[0], r[1], btag_WP)
        
        data_r   = datafile.Get(folder_r).Clone("data_"+r)
        data_r.SetDirectory(0)
        
        top_r    = topfile.Get(folder_r).Clone("top_"+r)
        top_r.SetDirectory(0)

        for ibin in range(1, top_r.GetNbinsX()+1):
            if top_r.GetBinContent(ibin) < 0:
                top_r.SetBinContent(ibin, 0)
                top_r.SetBinError(ibin, 0)
                

        histos[r]     = {"data": data_r,            "top": top_r}

    datafile.Close()
    topfile.Close()
    ##################################################################



    
    ##### scaling and subtractions #################################
    for ir in range(len(regions)):
        r = regions[ir]
        
        outfileStat = R.TFile("outfile_boosted_"+r+".root","RECREATE")
        
        r_2b = r[0]+"2"
        r_3b = r[0]+"3"

        top_2b = histos[r_2b]["top"].Clone("top_2b__"+r)
        if scaleTop2b:
            top_2b.Scale( (bkgFitResults["topscale"][0] if use_one_top_nuis else bkgFitResults["topscale"][ir]) )

        qcd_r = histos[r_2b]["data"].Clone("qcd__"+r)
        qcd_int = qcd_r.Integral()
        qcd_r.Add( top_2b, -1)      # added by Qi --- we still want top to be subtracted, given that their fraction is increasing in Run 2.

        top_r = histos[r]["top"].Clone("top__"+r)
        if (nbtag_top_shape =="3") and (r == "44"):   # the 3b top shape is only used during the SR prediction for 44 region
            temp_scaler = top_r.Integral() / histos[r_3b]["top"].Integral()
            top_r = histos[r_3b]["top"].Clone("top__"+r)
            top_r.Scale( temp_scaler )
        top_int = top_r.Integral()


        mu_qcd = bkgFitResults["muqcd"][ir]
        top_scale = (bkgFitResults["topscale"][0] if use_one_top_nuis else bkgFitResults["topscale"][ir])
        
        qcd_r.Scale( mu_qcd )
        top_r.Scale( top_scale )



        ## Now do smoothing

        qcd_sm = smoothfit.smoothfit(qcd_r, fitFunction = "Exp", fitRange = (900, 2000), makePlots = True, verbose = verbose, outfileName="qcd_smoothfit_"+r+".root")
        top_sm = smoothfit.smoothfit(top_r, fitFunction = "Exp", fitRange = (850, 1200), makePlots = True, verbose = verbose, outfileName="top_smoothfit_"+r+".root")

        qcd_final = smoothfit.MakeSmoothHisto(qcd_r, qcd_sm["nom"])
        top_final = smoothfit.MakeSmoothHisto(top_r, top_sm["nom"])

        if rebinFinal is not None:
            qcd_final = qcd_final.Rebin(len(rebinFinal)-1, qcd_final.GetName()+"_rebinFinal", rebinFinal)
            top_final = top_final.Rebin(len(rebinFinal)-1, top_final.GetName()+"_rebinFinal", rebinFinal)

        # outfileStat.WriteTObject(qcd_final, "qcd_hh_nominal","Overwrite")
        # outfileStat.WriteTObject(top_final, "top_hh_nominal","Overwrite")

        outfileStat.WriteTObject(qcd_final, "qcd_hh","Overwrite")
        outfileStat.WriteTObject(top_final, "ttbar_hh","Overwrite")

        

        ### propagate correlated systematics from the smoothing procedure---> these "replace" the stat error on the bins  #############
        for ivar in range(len(qcd_sm["vars"])):
            qup = qcd_sm["vars"][ivar][0]
            qdw = qcd_sm["vars"][ivar][1]

            qcd_r_qup = smoothfit.MakeSmoothHisto(qcd_r, qup)
            qcd_r_qdw = smoothfit.MakeSmoothHisto(qcd_r, qdw)

            if rebinFinal is not None:
                qcd_r_qup = qcd_r_qup.Rebin(len(rebinFinal)-1, qcd_r_qup.GetName()+"_rebinFinal", rebinFinal)
                qcd_r_qdw = qcd_r_qdw.Rebin(len(rebinFinal)-1, qcd_r_qdw.GetName()+"_rebinFinal", rebinFinal)

            outfileStat.WriteTObject(qcd_r_qup, "qcd_hh_smoothQ"+str(ivar)+"Up","Overwrite")
            outfileStat.WriteTObject(qcd_r_qdw, "qcd_hh_smoothQ"+str(ivar)+"Down","Overwrite")

        for ivar in range(len(top_sm["vars"])):
            tup = top_sm["vars"][ivar][0]
            tdw = top_sm["vars"][ivar][1]

            top_r_tup = smoothfit.MakeSmoothHisto(top_r, tup)
            top_r_tdw = smoothfit.MakeSmoothHisto(top_r, tdw)

            if rebinFinal is not None:
                top_r_tup = top_r_tup.Rebin(len(rebinFinal)-1, top_r_tup.GetName()+"_rebinFinal", rebinFinal)
                top_r_tdw = top_r_tdw.Rebin(len(rebinFinal)-1, top_r_tdw.GetName()+"_rebinFinal", rebinFinal)

            # outfileStat.WriteTObject(top_r_tup, "top_hh_smoothT"+str(ivar)+"Up","Overwrite")
            # outfileStat.WriteTObject(top_r_tdw, "top_hh_smoothT"+str(ivar)+"Down","Overwrite")

            outfileStat.WriteTObject(top_r_tup, "ttbar_hh_smoothT"+str(ivar)+"Up","Overwrite")
            outfileStat.WriteTObject(top_r_tdw, "ttbar_hh_smoothT"+str(ivar)+"Down","Overwrite")


            

        ### propagate correlated systematics from normalization fits for mu_qcd and top_scale ###############
        for ivar in range(len(pvars)):
            for iUD in range(2):
                mu_qcd_var = pvars[ivar][iUD][ir]
                top_scale_var = pvars[ivar][iUD][n_channels + (0 if use_one_top_nuis else ir) ]

                qvar = qcd_r.Clone("qvar")
                qvar.Scale( mu_qcd_var * qcd_int / qvar.Integral() )

                tvar = top_r.Clone("tvar")
                tvar.Scale( top_scale_var * top_int / tvar.Integral() )

                ## Now do smoothing

                qvar_sm = smoothfit.smoothfit(qvar, fitFunction = "Exp", fitRange = (900, 2000), makePlots = False, verbose = verbose,
                                              outfileName="qcd_smoothfit_"+r+"_Norm"+str(ivar)+str(iUD)+".root")
                tvar_sm = smoothfit.smoothfit(tvar, fitFunction = "Exp", fitRange = (850, 1200), makePlots = False, verbose = verbose,
                                              outfileName="top_smoothfit_"+r+"_Norm"+str(ivar)+str(iUD)+".root")
    
                qvar_final = smoothfit.MakeSmoothHisto(qvar, qvar_sm["nom"])
                tvar_final = smoothfit.MakeSmoothHisto(tvar, tvar_sm["nom"])

                if rebinFinal is not None:
                    qvar_final = qvar_final.Rebin(len(rebinFinal)-1, qvar_final.GetName()+"_rebinFinal", rebinFinal)
                    tvar_final = tvar_final.Rebin(len(rebinFinal)-1, tvar_final.GetName()+"_rebinFinal", rebinFinal)

                UpDw = ("Up" if iUD ==0 else "Down")
                outfileStat.WriteTObject(qvar_final, "qcd_hh_normY"+str(ivar)+UpDw,"Overwrite")
                # outfileStat.WriteTObject(tvar_final, "top_hh_normY"+str(ivar)+UpDw,"Overwrite")
                outfileStat.WriteTObject(tvar_final, "ttbar_hh_normY"+str(ivar)+UpDw,"Overwrite")
            
        
        
        outfileStat.Close()
        

        

        ## if False:
        ##     pred_final = qcd_final.Clone("pred_final__"+r)
        ##     pred_final.Add( top_final )


        ##     func1 = qcd_sm["nom"]
        ##     func2 = top_sm["nom"]

        ##     pred_sm = R.TF1("pred_sm", FuncSum, 900, 3000)

        ##     pred_sm.Draw("same")
        ##     top_sm["nom"].Draw("same")

        ##     pred_final_raw = qcd_r.Clone("qcd_final_raw__"+r)
        ##     pred_final_raw.Add(top_r)

        ##     outfile = R.TFile("outfile_"+r+".root","RECREATE")

        ##     c=R.TCanvas()
        ##     pred_final_raw.Draw("HIST")
        ##     top_r.SetLineColor(R.kBlack)
        ##     top_r.SetFillColor(R.kGreen)
        ##     top_r.Draw("sameHIST")

        ##     pred_sm.Draw("same")
        ##     top_sm["nom"].Draw("same")

        ##     c.Write()

        ##     c=R.TCanvas()

        ##     pred_final.Draw("HIST")

        ##     top_final.SetLineColor(R.kBlack)
        ##     top_final.SetFillColor(R.kGreen)

        ##     top_final.Draw("sameHIST")

        ##     c.Write()

        ##     outfile.Close()

    


    return