def main(argv):
    args = parseArgs(argv)
    for run in range(args.num):

        # Read input files
        aos = yoda.read(args.dir + os.path.sep + str(run).zfill(4) +
                        os.path.sep + args.yodafile)
        fpar = open(
            args.dir + os.path.sep + str(run).zfill(4) + os.path.sep +
            args.paramfile, 'r')
        param = {}
        for line in fpar:
            (k, v) = line.split()
            param[k.strip()] = v.strip()
        fpar.close()

        # Extract scatterplots from yoda file
        scatters = []
        for aopath, ao in aos.iteritems():
            scatterplot = ao.mkScatter()
            scatterplot.setAnnotation("Run_Directory", args.dir)
            for key, value in param.iteritems():
                scatterplot.setAnnotation("Tune_Parameter_" + key, value)

            # Check for patterns and unpatterns
            save = True
            if args.patterns != None:
                save = False
                for pattern in args.patterns:
                    if pattern in scatterplot.path:
                        save = True
            if args.unpatterns != None:
                for unpattern in args.unpatterns:
                    if unpattern in scatterplot.path:
                        save = False
            if save:
                scatters.append((aopath, scatterplot))

        yoda.writeYODA(dict(scatters),
                       "run" + str(run).zfill(4) + "_" + args.yodafile)
# get hold of relevant objects in reference data files
refhistos = getRivetRefData(['ATLAS_2017_I1609448'])

def constructRmiss(hist):
    '''This recreates the constructRmiss function from the routine.'''
    rtn = yoda.mkScatter(hist)
    path = hist.annotation('Path').replace('_d', 'd')
    numer = refhistos[path.replace('y02', 'y03')]
    denom = refhistos[path.replace('y02', 'y04')]
    rmiss = refhistos[path]
    for i in range(rtn.numPoints):
        newy = (numer.points[i].y + rtn.points[i].y) / denom.points[i].y if denom.points[i].y else 0.0
        # ratio error (Rmiss = SM_num/SM_denom + BSM/SM_denom ~ Rmiss_SM + BSM/SM_denom
        rel_hist_err = rtn.points[i].yErrs[0] / denom.points[i].y if denom.points[i].y else 0.0
        newey = sqrt(rmiss.points[i].yErrs[0] ** 2 + rel_hist_err ** 2)
        rtn.points[i].y = newy
        rtn.points[i].yErrs = (newey, newey)
    return rtn

# this is where the magic happens
f = open('%s_processed.yoda' % inFile[:-5], 'w')
for h in tags:
    if 'y02' in h:
        outName = h.replace('_d', 'd')
        outName = outName.replace('y02', 'y01')
        rmiss = constructRmiss(hists[h])
        rmiss.setAnnotation('Path', outName)
        yoda.writeYODA(rmiss, f)
f.close()

      output_object.normalize(normto)

  return output_object

parser = optparse.OptionParser(usage=__doc__)
parser.add_option('-o', '--output', default='-', dest='OUTPUT_FILE')
parser.add_option('-N', '--normalize-all', action="store_true", default=False, dest='NORMALIZE_ALL')
opts, fileargs = parser.parse_args()

## Put the incoming objects into a dict from each path to a list of histos and scalings
analysisobjects_in = {}
for fa in fileargs:
    filename, scale = fa, 1.0 
    if ":" in fa: 
        try:
            filename, scale = fa.rsplit(":", 1)
            scale = float(scale)
        except:
            sys.stderr.write("Error processing arg '%s' with file:scale format\n" % fa) 
    aos = yoda.read(filename)
    for aopath, ao in aos.iteritems():
        ao.setAnnotation("yodamerge_scale", scale)
        analysisobjects_in.setdefault(aopath, []).append(ao)

analysisobjects_out = {}
for path, aos in analysisobjects_in.iteritems():
  if(path in EFFICIENCIES): analysisobjects_out[path] = MergeEfficiency(path, aos)
  else: analysisobjects_out[path] = MergeDistribution(path, aos)

yoda.writeYODA(analysisobjects_out, opts.OUTPUT_FILE)
Пример #4
0
#! /usr/bin/env python

from sys import stdout, argv
import yoda

analysisobjects_in = {}
for filename in argv[1:]:
    aos = yoda.read(filename)
    for aopath, ao in aos.iteritems():
        analysisobjects_in.setdefault(aopath, []).append(ao)


analysisobjects_out = {}
for p, aos in analysisobjects_in.iteritems():
    ao_out = aos[0].clone()
    for ao in aos[1:]:
        ao_out += ao

    analysisobjects_out[p] = ao_out

yoda.writeYODA(analysisobjects_out, stdout)
def constructRmiss(hist):
    '''This recreates the constructRmiss function from the routine.'''
    rtn = yoda.mkScatter(hist)
    path = hist.annotation('Path').replace('_d', 'd')
    numer = refhistos[path.replace('y02', 'y03')]
    denom = refhistos[path.replace('y02', 'y04')]
    rmiss = refhistos[path]
    for i in range(rtn.numPoints):
        newy = (numer.points[i].y + rtn.points[i].y
                ) / denom.points[i].y if denom.points[i].y else 0.0
        # ratio error (Rmiss = SM_num/SM_denom + BSM/SM_denom ~ Rmiss_SM + BSM/SM_denom
        rel_hist_err = rtn.points[i].yErrs[0] / denom.points[
            i].y if denom.points[i].y else 0.0
        newey = sqrt(rmiss.points[i].yErrs[0]**2 + rel_hist_err**2)
        rtn.points[i].y = newy
        rtn.points[i].yErrs = (newey, newey)
    return rtn


# this is where the magic happens
f = open('%s_processed.yoda' % inFile[:-5], 'w')
for h in tags:
    if 'y02' in h:
        outName = h.replace('_d', 'd')
        outName = outName.replace('y02', 'y01')
        rmiss = constructRmiss(hists[h])
        rmiss.setAnnotation('Path', outName)
        yoda.writeYODA(rmiss, f)
f.close()
Пример #6
0
#!/usr/bin/env python

import yoda,sys,re

if len(sys.argv) != 3:
    sys.exit("Usage: yoda-rebin.py input.yoda output.yoda")

all_aos = yoda.readYODA(sys.argv[1])

for ao in all_aos.itervalues():
    # we want to rebin anything except multiplicity
    pattern = re.compile(".*GA_00_00.*|.*Mult.*")

    if not pattern.match(ao.path):
        ao.rebin(5)

yoda.writeYODA(all_aos,sys.argv[2])
Пример #7
0
        'd03', 'd04')  # data statistical uncertainty
    path_unco = hist.annotation('Path').replace(
        'd03', 'd05')  # data uncorrelated uncertainty
    data = refhistos[path_data]
    stat = refhistos[path_stat]
    unco = refhistos[path_unco]
    data_integral = sum([p.z for p in data.points])
    hist.normalize(data_integral)
    rtn = hist.clone()
    rtn.reset()
    mc = yoda.mkScatter(hist)
    for i in range(rtn.numBins):
        sigma = sqrt(stat.points[i].z**2 + unco.points[i].z**2 +
                     (mc.points[i].zErrs[0])**2)
        newz = (data.points[i].z - mc.points[i].z) / sigma if sigma else 0.0
        #newz = (0.01 * mc.points[i].z - data.points[i].z) / sigma if sigma else 0.0
        rtn.fillBin(i, newz)
    return rtn


# this is where the magic happens
f = open('%s_processed.yoda' % inFile[:-5], 'w')
for h in tags:
    if 'd03-x01-y01' in h:
        hdiff = constructDiff(hists[h])
        outName = h.replace('y01', 'y02')
        hdiff.setAnnotation('Path', outName)
        yoda.writeYODA(hdiff, f)
    yoda.writeYODA(hists[h], f)
f.close()
Пример #8
0
def constructDiff(hist):
    '''This function produces a (data - MC)/sigma version of the Dalitz (2D) plot.'''
    path_data = hist.annotation('Path') # data central value
    path_stat = hist.annotation('Path').replace('d03', 'd04') # data statistical uncertainty
    path_unco = hist.annotation('Path').replace('d03', 'd05') # data uncorrelated uncertainty
    data = refhistos[path_data]
    stat = refhistos[path_stat]
    unco = refhistos[path_unco]
    data_integral = sum([ p.z for p in data.points ])
    hist.normalize(data_integral) 
    rtn = hist.clone();  rtn.reset()
    mc = yoda.mkScatter(hist)
    for i in range(rtn.numBins):
        sigma = sqrt(stat.points[i].z ** 2 + unco.points[i].z ** 2 + (mc.points[i].zErrs[0]) ** 2)
        newz = (data.points[i].z - mc.points[i].z) / sigma if sigma else 0.0
        #newz = (0.01 * mc.points[i].z - data.points[i].z) / sigma if sigma else 0.0
        rtn.fillBin(i, newz)
    return rtn

# this is where the magic happens
f = open('%s_processed.yoda' % inFile[:-5], 'w')
for h in tags:
    if 'd03-x01-y01' in h:
        hdiff = constructDiff(hists[h])
        outName = h.replace('y01', 'y02')
        hdiff.setAnnotation('Path', outName)
        yoda.writeYODA(hdiff, f)
    yoda.writeYODA(hists[h], f)
f.close()

def main():
    CommonFSQFramework.Core.Style.setTDRStyle()


    parser = OptionParser(usage="usage: %prog [options] filename",
                            version="%prog 1.0")

    parser.add_option("-v", "--variant",   action="store", dest="variant", type="string", \
                                help="choose analysis variant")
    parser.add_option("-n", "--normalization",   action="store", dest="normalization", type="string", \
                                help="how should I normalize the plots?")
    parser.add_option("-b", "--normalizeToBinWidth",   action="store_true", dest="normalizeToBinWidth")
    parser.add_option("-s", type="float", dest="scaleExtra")   

    (options, args) = parser.parse_args()
    scaleExtra = 1.
    if options.scaleExtra:
        scaleExtra = 1./options.scaleExtra

    normalizeToBinWidth = False
    if options.normalizeToBinWidth:
        normalizeToBinWidth = True

    if not options.variant:
        print "Provide analysis variant"
        sys.exit()

    if not options.normalization:
        print "Provide normalization variant"
        sys.exit()

    norms = ["xs", "area"]
    if options.normalization not in norms:
        print "Normalization not known. Possible choices: " + " ".join(norms)
        sys.exit()

    indir = "~/tmp/unfolded_{}/".format(options.variant)


    (options, args) = parser.parse_args()
    if not options.variant:
        print "Provide analysis variant"
        sys.exit()

    indir = "~/tmp/unfolded_{}/".format(options.variant)
    histofile = "plotsMNxs_{}.root".format(options.variant)


    lumiUncertainty = 0.04
    herwigIn=indir+"/mnxsHistos_unfolded_herwigOnData.root"
    pythiaIn=indir+"/mnxsHistos_unfolded_pythiaOnData.root"
    ofileName = indir+"/mnxsHistos_unfolded_onData_merged.root"


    histos = {}
    histos["herwig"]=getHistos(herwigIn)
    histos["pythia"]=getHistos(pythiaIn)
    #print histos["herwig"]["_jet15"].keys()
    #sys.exit()
    # TODO: test that dirs have the same contents

    # ['xsunfolded_central_jet15', 'xsunfolded_jecDown_jet15', 'xs_central_jet15', 'xsunfolded_jerDown_jet15', 'xsunfolded_jecUp_jet15', 'xsunfolded_jerUp_jet15']
    finalSet = {}

    todo = ["_jet15", "_dj15fb"]
    #todo = ["_jet15"]
    for t in todo:
        finalSet[t] = {}
        for hName in histos["herwig"][t]:
            if hName.startswith("xs_"): continue # skip detector level histogram

            hAvg = histos["herwig"][t][hName].Clone()

            hAvg.Add(histos["pythia"][t][hName])
            hAvg.Scale(0.5)
            finalSet[t][hName]=hAvg

            # add herwig/pythia central histo as variations
            #  in case we would have more than two MC - for every MC
            #   add a central value as "up" variation, as a "down"
            #   variation use the averaged histogram
            #    this way we have consistent list of up/down variations,
            #    where the down variation doesnt enlarge uncertainty band
            if "_central_" in hName:
                newNameHerwig = hName.replace("_central_", "_modelUp_")
                newNamePythia = hName.replace("_central_", "_modelDown_")
                finalSet[t][newNameHerwig] = histos["herwig"][t][hName].Clone(newNameHerwig)
                finalSet[t][newNamePythia] = histos["pythia"][t][hName].Clone(newNamePythia)

                # at the same point - use the averaged histogram to add lumi uncertainy
                #  BTW: should we do it here??
                newNameAvgUp = hName.replace("_central_", "_lumiUp_")
                newNameAvgDown = hName.replace("_central_", "_lumiDown_")
                finalSet[t][newNameAvgUp] = hAvg.Clone(newNameAvgUp)
                finalSet[t][newNameAvgDown] = hAvg.Clone(newNameAvgDown)
                finalSet[t][newNameAvgUp].Scale(1.+lumiUncertainty)
                finalSet[t][newNameAvgDown].Scale(1.-lumiUncertainty)



    # add jet15 and dj15 histos
    # note: histo binning should be the same from beginning!
    finalSet["merged"] = {}
    for t in finalSet["_jet15"]:
        newName = t.replace("_jet15", "_jet15andDJ15FB")
        finalHisto = finalSet["_jet15"][t].Clone(newName)
        finalHisto.Add(finalSet["_dj15fb"][t.replace("_jet15", "_dj15fb")].Clone())
        if options.normalization == "area":
            finalHisto.Scale(1./finalHisto.Integral())
        if normalizeToBinWidth:
            finalHisto.Scale(1., "width")

        finalHisto.Scale(scaleExtra)

        finalSet["merged"][newName] = finalHisto
            



    # save all to file
    ofile = ROOT.TFile(ofileName, "RECREATE")
    for dirName in finalSet:
        odir = ofile.mkdir(dirName)
        for h in finalSet[dirName]:
            odir.WriteTObject(finalSet[dirName][h])


    # make final plot, including uncertainty band
    central = [ finalSet["merged"][hName] for hName in finalSet["merged"].keys() if "_central_" in hName ]
    if len(central) != 1:
        raise Exception("Error: more than one central histo found")
    central = central[0]

    uncert  = [finalSet["merged"][hName] for hName in finalSet["merged"].keys() if "_central_" not in hName ]
    #uncert  = [finalSet["merged"][hName] for hName in finalSet["merged"].keys() if "_model" in hName ]

    uncResult= DrawPlots.getUncertaintyBand(uncert, central)
    unc = uncResult["band"]


    # get GEN level distributions
    histosFromPyAnalyzer = getHistos(histofile)
    herwigDir = "QCD_Pt-15to1000_TuneEE3C_Flat_7TeV_herwigpp"
    pythiaDir =  "QCD_Pt-15to3000_TuneZ2star_Flat_HFshowerLibrary_7TeV_pythia6"
    genHistoHerwig = histosFromPyAnalyzer[herwigDir]["detaGen_central_jet15"].Clone()
    genHistoHerwig.Add(histosFromPyAnalyzer[herwigDir]["detaGen_central_dj15fb"])
    genHistoPythia = histosFromPyAnalyzer[pythiaDir]["detaGen_central_jet15"].Clone()
    genHistoPythia.Add(histosFromPyAnalyzer[pythiaDir]["detaGen_central_dj15fb"])

    if options.normalization == "area":
        map(lambda h: h.Scale(1./h.Integral()), [genHistoPythia, genHistoHerwig] )
    if normalizeToBinWidth:
        map(lambda h: h.Scale(1, "width"), [genHistoPythia, genHistoHerwig] )
    map(lambda h: h.Scale(scaleExtra), [genHistoPythia, genHistoHerwig] )

    maxima = []
    maxima.append(uncResult["max"])
    for t in [unc, central, genHistoHerwig, genHistoPythia]:
        maxima.append(t.GetMaximum())

    c = ROOT.TCanvas()
    c.Divide(1,2)
    c.cd(1)
    split = 0.2
    margin = 0.005
    ROOT.gPad.SetPad(.005, split+margin, .995, .995)
    c.cd(2)
    ROOT.gPad.SetPad(.005, .005, .995, split)
    c.cd(1)

    ROOT.gPad.SetTopMargin(0.1)
    #c.SetRightMargin(0.07)
    central.SetMaximum(max(maxima)*1.05)
    unc.SetFillColor(17);
    central.Draw()
    #central.GetXaxis().SetRangeUser(5,8)
    #central.GetYaxis().SetRangeUser(0,250000)

    central.GetXaxis().SetTitle("#Delta#eta")
    central.GetYaxis().SetTitle("#sigma [pb]")
    central.GetYaxis().SetTitleOffset(1.8)
    unc.Draw("2SAME")
    central.Draw("SAME")

    genHistoHerwig.Draw("SAME HIST")
    genHistoHerwig.SetLineColor(2)

    genHistoPythia.Draw("SAME HIST")
    genHistoPythia.SetLineColor(4)

    DrawMNPlots.banner()


    legend = ROOT.TLegend(0.6, 0.7, 0.9, 0.85)
    legend.SetFillColor(0)
    legend.AddEntry(central, "data", "pel")
    legend.AddEntry(unc, "syst. unc.", "f")
    legend.AddEntry(genHistoHerwig, "herwig", "l")
    legend.AddEntry(genHistoPythia, "pythia", "l")
    legend.Draw("SAME")    

    c.cd(2)
    frame = ROOT.gPad.DrawFrame(central.GetXaxis().GetXmin(), 0, central.GetXaxis().GetXmax(), 3)
    #frame.GetXaxis().SetRangeUser(5,8)

    yUp = array('d')
    yDown = array('d')
    x = array('d')
    y = array('d')
    xDown = array('d')
    xUp = array('d')

    y4Rivet = array('d')
    yUp4Rivet = array('d')
    yDown4Rivet = array('d')
    for iBin in xrange(1, central.GetNbinsX()+1):
        val =  central.GetBinContent(iBin)
        if val == 0: continue

        if val != 0:
            binErr  = central.GetBinError(iBin)
            errUp = unc.GetErrorYhigh(iBin-1)
            errDown =  unc.GetErrorYlow(iBin-1)
            valDown = errDown/val
            valUp =   errUp/val
            yDown.append(valDown)
            yUp.append(valUp)
            valDown4Rivet = math.sqrt(errDown*errDown + binErr*binErr  )
            valUp4Rivet   = math.sqrt(errUp*errUp + binErr*binErr  )
            yUp4Rivet.append(valUp4Rivet)
            yDown4Rivet.append(valDown4Rivet)
            #print valDown, valUp
        else:
           yUp.append(0)
           yDown.append(0)
        #print 
        x.append(unc.GetX()[iBin-1])
        y.append(1)
        ratio = unc.GetY()[iBin-1]/val
        if max(ratio-1., 1.-ratio)>0.001:
            raise Exception("Expected equal values")

        y4Rivet.append(val)
        xDown.append(unc.GetErrorXlow(iBin-1))
        xUp.append(unc.GetErrorXhigh(iBin-1))

    #print type(x)
    uncRatio =     ROOT.TGraphAsymmErrors(len(x), x, y, xDown, xUp, yDown, yUp)
    result4Rivet = ROOT.TGraphAsymmErrors(len(x), x, y4Rivet, xDown, xUp, yDown4Rivet, yUp4Rivet)

    #uncRatio = ROOT.TGraphAsymmErrors(len(x), x, y, xDown, xUp, yDown, yUp)

    uncRatio.SetFillStyle(3001)
    uncRatio.SetFillColor(17)
    uncRatio.Draw("2SAME")


    centralRatio = central.Clone()
    centralRatio.Divide(central)
    centralRatio.Draw("SAME")

    herwigRatio = genHistoHerwig.Clone()
    herwigRatio.Divide(central)

    pythiaRatio = genHistoPythia.Clone()
    pythiaRatio.Divide(central)

    herwigRatio.Draw("SAME L")
    pythiaRatio.Draw("SAME L")


    c.Print(indir+"/mergedUnfolded_{}.png".format(options.normalization))
    c.Print(indir+"/mergedUnfolded_{}.pdf".format(options.normalization))
    c.Print(indir+"/mergedUnfolded_{}.root".format(options.normalization))
    c.cd(1)
    ROOT.gPad.SetLogy()
    c.Print(indir+"/mergedUnfolded_{}_log.png".format(options.normalization))
    c.Print(indir+"/mergedUnfolded_{}_log.pdf".format(options.normalization))


    # rivet export
    from do import todoCatAll
    if len(todoCatAll) != 6:
        raise Exception("Error: inconsistent number of categories in todoCatAll")
    rivet = ROOT.TFile("toRivet.root", "RECREATE")
    rivetNum = todoCatAll.index(options.variant)+1
    if "area" == options.normalization:
        rivetNum += 10
    numAsStr = str(rivetNum)
    if len (numAsStr) == 1:
        numAsStr = "0"+numAsStr

    rivetName = "d"+numAsStr+"-x01-y01"
    print options.normalization, rivetNum, rivetName
    rivet.WriteTObject(result4Rivet, rivetName)
    rivet.Close()
    del rivet

    import os
    r2f = "/cvmfs/cms.cern.ch/slc6_amd64_gcc481/external/rivet/1.8.2-cms8/bin/root2flat"
    if not os.path.isfile(r2f):
        raise Exception("Cannot find root2flat. Rivet export failed")

    os.system(r2f + " toRivet.root")

    import yoda
    analysisobjects = yoda.readFLAT(rivetName+".dat")
    #print type(analysisobjects)
    #print analysisobjects.keys()
    for k in analysisobjects:
        pth = "/CMS_2015_FWD071/"+rivetName
        #print dir(analysisobjects[k])
        #analysisobjects[k].setTitle(pth)
        #analysisobjects[k].setPath(pth)
        analysisobjects[k].setAnnotation("Title", pth)
        analysisobjects[k].setAnnotation("Path",  pth)

    yoda.writeYODA(analysisobjects, rivetName+".yoda")
    all_sep.append(obs_sep)
        
    # overflow contribution:
    wq=1-wtotq
    if (wq<0): wq=0
    wg=1-wtotg
    if (wg<0): wg=0
    
    I05_second += (0.5*(wq-wg)*(wq-wg)/(wq+wg) if wq+wg>0 else 0.0)
    
    I05+=(0.5*wq*math.log(2*wq/(wq+wg))/math.log(2.0) if wq>0 else 0.0)
    I05+=(0.5*wg*math.log(2*wg/(wq+wg))/math.log(2.0) if wg>0 else 0.0)
    
    print '{0:30s} {1:8.4f} {2:8.4f} {3:8.4f} {4:8.4f} {5:8.4f} {6:8.4f} {7:8.4f}'.format(label,grej20,grej50,qrej20,qrej50,srej,I05,I05_second)

yoda.writeYODA(all_sep,sys.argv[3])
    
    
    # # file header
    # lineq = read_until_matching(fileq, re.compile("^BEGIN YODA_HISTO1D /MC_LHQG|^# BEGIN YODA_HISTO1D /MC_LHQG"));
    # lineg = read_until_matching(fileg, re.compile("^BEGIN YODA_HISTO1D /MC_LHQG|^# BEGIN YODA_HISTO1D /MC_LHQG"));
    # print lineq
    # print lineg
    # fileo.write(lineq);
    # lineq = read_until_matching(fileq, re.compile("^Path="));
    # print lineq
    # #lineq=lineq.replace("Path=/MC_LHQG_EE/","")
    # #lineq=lineq.replace("Path=/MC_LHQG_dijet/","")
    # #label=lineq.replace("Path=/MC_LHQG_Zjet/","").rstrip("\n")
    # #fileo.write(lineq)
    # fileo.write(lineq)
Пример #11
0
#!/usr/bin/env python

import yoda, sys, re

if len(sys.argv) != 3:
    sys.exit("Usage: yoda-rebin.py input.yoda output.yoda")

all_aos = yoda.readYODA(sys.argv[1])

for ao in all_aos.itervalues():
    # we want to rebin anything except multiplicity
    pattern = re.compile(".*GA_00_00.*|.*Mult.*")

    if not pattern.match(ao.path):
        ao.rebin(5)

yoda.writeYODA(all_aos, sys.argv[2])
def main():
    CommonFSQFramework.Core.Style.setTDRStyle()

    parser = OptionParser(usage="usage: %prog [options] filename",
                          version="%prog 1.0")

    parser.add_option("-v", "--variant",   action="store", dest="variant", type="string", \
                                help="choose analysis variant")
    parser.add_option("-n", "--normalization",   action="store", dest="normalization", type="string", \
                                help="how should I normalize the plots?")
    parser.add_option("-b",
                      "--normalizeToBinWidth",
                      action="store_true",
                      dest="normalizeToBinWidth")
    parser.add_option("-s", type="float", dest="scaleExtra")

    (options, args) = parser.parse_args()
    scaleExtra = 1.
    if options.scaleExtra:
        scaleExtra = 1. / options.scaleExtra

    normalizeToBinWidth = False
    if options.normalizeToBinWidth:
        normalizeToBinWidth = True

    if not options.variant:
        print "Provide analysis variant"
        sys.exit()

    if not options.normalization:
        print "Provide normalization variant"
        sys.exit()

    norms = ["xs", "area"]
    if options.normalization not in norms:
        print "Normalization not known. Possible choices: " + " ".join(norms)
        sys.exit()

    indir = "~/tmp/unfolded_{}/".format(options.variant)

    (options, args) = parser.parse_args()
    if not options.variant:
        print "Provide analysis variant"
        sys.exit()

    indir = "~/tmp/unfolded_{}/".format(options.variant)
    histofile = "plotsMNxs_{}.root".format(options.variant)

    lumiUncertainty = 0.04
    herwigIn = indir + "/mnxsHistos_unfolded_herwigOnData.root"
    pythiaIn = indir + "/mnxsHistos_unfolded_pythiaOnData.root"
    ofileName = indir + "/mnxsHistos_unfolded_onData_merged.root"

    histos = {}
    histos["herwig"] = getHistos(herwigIn)
    histos["pythia"] = getHistos(pythiaIn)
    #print histos["herwig"]["_jet15"].keys()
    #sys.exit()
    # TODO: test that dirs have the same contents

    # ['xsunfolded_central_jet15', 'xsunfolded_jecDown_jet15', 'xs_central_jet15', 'xsunfolded_jerDown_jet15', 'xsunfolded_jecUp_jet15', 'xsunfolded_jerUp_jet15']
    finalSet = {}

    todo = ["_jet15", "_dj15fb"]
    #todo = ["_jet15"]
    for t in todo:
        finalSet[t] = {}
        for hName in histos["herwig"][t]:
            if hName.startswith("xs_"):
                continue  # skip detector level histogram

            hAvg = histos["herwig"][t][hName].Clone()

            hAvg.Add(histos["pythia"][t][hName])
            hAvg.Scale(0.5)
            finalSet[t][hName] = hAvg

            # add herwig/pythia central histo as variations
            #  in case we would have more than two MC - for every MC
            #   add a central value as "up" variation, as a "down"
            #   variation use the averaged histogram
            #    this way we have consistent list of up/down variations,
            #    where the down variation doesnt enlarge uncertainty band
            if "_central_" in hName:
                newNameHerwig = hName.replace("_central_", "_modelUp_")
                newNamePythia = hName.replace("_central_", "_modelDown_")
                finalSet[t][newNameHerwig] = histos["herwig"][t][hName].Clone(
                    newNameHerwig)
                finalSet[t][newNamePythia] = histos["pythia"][t][hName].Clone(
                    newNamePythia)

                # at the same point - use the averaged histogram to add lumi uncertainy
                #  BTW: should we do it here??
                newNameAvgUp = hName.replace("_central_", "_lumiUp_")
                newNameAvgDown = hName.replace("_central_", "_lumiDown_")
                finalSet[t][newNameAvgUp] = hAvg.Clone(newNameAvgUp)
                finalSet[t][newNameAvgDown] = hAvg.Clone(newNameAvgDown)
                finalSet[t][newNameAvgUp].Scale(1. + lumiUncertainty)
                finalSet[t][newNameAvgDown].Scale(1. - lumiUncertainty)

    # add jet15 and dj15 histos
    # note: histo binning should be the same from beginning!
    finalSet["merged"] = {}
    for t in finalSet["_jet15"]:
        newName = t.replace("_jet15", "_jet15andDJ15FB")
        finalHisto = finalSet["_jet15"][t].Clone(newName)
        finalHisto.Add(finalSet["_dj15fb"][t.replace("_jet15",
                                                     "_dj15fb")].Clone())
        if options.normalization == "area":
            finalHisto.Scale(1. / finalHisto.Integral())
        if normalizeToBinWidth:
            finalHisto.Scale(1., "width")

        finalHisto.Scale(scaleExtra)

        finalSet["merged"][newName] = finalHisto

    # save all to file
    ofile = ROOT.TFile(ofileName, "RECREATE")
    for dirName in finalSet:
        odir = ofile.mkdir(dirName)
        for h in finalSet[dirName]:
            odir.WriteTObject(finalSet[dirName][h])

    # make final plot, including uncertainty band
    central = [
        finalSet["merged"][hName] for hName in finalSet["merged"].keys()
        if "_central_" in hName
    ]
    if len(central) != 1:
        raise Exception("Error: more than one central histo found")
    central = central[0]

    uncert = [
        finalSet["merged"][hName] for hName in finalSet["merged"].keys()
        if "_central_" not in hName
    ]
    #uncert  = [finalSet["merged"][hName] for hName in finalSet["merged"].keys() if "_model" in hName ]

    uncResult = DrawPlots.getUncertaintyBand(uncert, central)
    unc = uncResult["band"]

    # get GEN level distributions
    histosFromPyAnalyzer = getHistos(histofile)
    herwigDir = "QCD_Pt-15to1000_TuneEE3C_Flat_7TeV_herwigpp"
    pythiaDir = "QCD_Pt-15to3000_TuneZ2star_Flat_HFshowerLibrary_7TeV_pythia6"
    genHistoHerwig = histosFromPyAnalyzer[herwigDir][
        "detaGen_central_jet15"].Clone()
    genHistoHerwig.Add(
        histosFromPyAnalyzer[herwigDir]["detaGen_central_dj15fb"])
    genHistoPythia = histosFromPyAnalyzer[pythiaDir][
        "detaGen_central_jet15"].Clone()
    genHistoPythia.Add(
        histosFromPyAnalyzer[pythiaDir]["detaGen_central_dj15fb"])

    if options.normalization == "area":
        map(lambda h: h.Scale(1. / h.Integral()),
            [genHistoPythia, genHistoHerwig])
    if normalizeToBinWidth:
        map(lambda h: h.Scale(1, "width"), [genHistoPythia, genHistoHerwig])
    map(lambda h: h.Scale(scaleExtra), [genHistoPythia, genHistoHerwig])

    maxima = []
    maxima.append(uncResult["max"])
    for t in [unc, central, genHistoHerwig, genHistoPythia]:
        maxima.append(t.GetMaximum())

    c = ROOT.TCanvas()
    c.Divide(1, 2)
    c.cd(1)
    split = 0.2
    margin = 0.005
    ROOT.gPad.SetPad(.005, split + margin, .995, .995)
    c.cd(2)
    ROOT.gPad.SetPad(.005, .005, .995, split)
    c.cd(1)

    ROOT.gPad.SetTopMargin(0.1)
    #c.SetRightMargin(0.07)
    central.SetMaximum(max(maxima) * 1.05)
    unc.SetFillColor(17)
    central.Draw()
    #central.GetXaxis().SetRangeUser(5,8)
    #central.GetYaxis().SetRangeUser(0,250000)

    central.GetXaxis().SetTitle("#Delta#eta")
    central.GetYaxis().SetTitle("#sigma [pb]")
    central.GetYaxis().SetTitleOffset(1.8)
    unc.Draw("2SAME")
    central.Draw("SAME")

    genHistoHerwig.Draw("SAME HIST")
    genHistoHerwig.SetLineColor(2)

    genHistoPythia.Draw("SAME HIST")
    genHistoPythia.SetLineColor(4)

    DrawMNPlots.banner()

    legend = ROOT.TLegend(0.6, 0.7, 0.9, 0.85)
    legend.SetFillColor(0)
    legend.AddEntry(central, "data", "pel")
    legend.AddEntry(unc, "syst. unc.", "f")
    legend.AddEntry(genHistoHerwig, "herwig", "l")
    legend.AddEntry(genHistoPythia, "pythia", "l")
    legend.Draw("SAME")

    c.cd(2)
    frame = ROOT.gPad.DrawFrame(central.GetXaxis().GetXmin(), 0,
                                central.GetXaxis().GetXmax(), 3)
    #frame.GetXaxis().SetRangeUser(5,8)

    yUp = array('d')
    yDown = array('d')
    x = array('d')
    y = array('d')
    xDown = array('d')
    xUp = array('d')

    y4Rivet = array('d')
    yUp4Rivet = array('d')
    yDown4Rivet = array('d')
    for iBin in xrange(1, central.GetNbinsX() + 1):
        val = central.GetBinContent(iBin)
        if val == 0: continue

        if val != 0:
            binErr = central.GetBinError(iBin)
            errUp = unc.GetErrorYhigh(iBin - 1)
            errDown = unc.GetErrorYlow(iBin - 1)
            valDown = errDown / val
            valUp = errUp / val
            yDown.append(valDown)
            yUp.append(valUp)
            valDown4Rivet = math.sqrt(errDown * errDown + binErr * binErr)
            valUp4Rivet = math.sqrt(errUp * errUp + binErr * binErr)
            yUp4Rivet.append(valUp4Rivet)
            yDown4Rivet.append(valDown4Rivet)
            #print valDown, valUp
        else:
            yUp.append(0)
            yDown.append(0)
        #print
        x.append(unc.GetX()[iBin - 1])
        y.append(1)
        ratio = unc.GetY()[iBin - 1] / val
        if max(ratio - 1., 1. - ratio) > 0.001:
            raise Exception("Expected equal values")

        y4Rivet.append(val)
        xDown.append(unc.GetErrorXlow(iBin - 1))
        xUp.append(unc.GetErrorXhigh(iBin - 1))

    #print type(x)
    uncRatio = ROOT.TGraphAsymmErrors(len(x), x, y, xDown, xUp, yDown, yUp)
    result4Rivet = ROOT.TGraphAsymmErrors(len(x), x, y4Rivet, xDown, xUp,
                                          yDown4Rivet, yUp4Rivet)

    #uncRatio = ROOT.TGraphAsymmErrors(len(x), x, y, xDown, xUp, yDown, yUp)

    uncRatio.SetFillStyle(3001)
    uncRatio.SetFillColor(17)
    uncRatio.Draw("2SAME")

    centralRatio = central.Clone()
    centralRatio.Divide(central)
    centralRatio.Draw("SAME")

    herwigRatio = genHistoHerwig.Clone()
    herwigRatio.Divide(central)

    pythiaRatio = genHistoPythia.Clone()
    pythiaRatio.Divide(central)

    herwigRatio.Draw("SAME L")
    pythiaRatio.Draw("SAME L")

    c.Print(indir + "/mergedUnfolded_{}.png".format(options.normalization))
    c.Print(indir + "/mergedUnfolded_{}.pdf".format(options.normalization))
    c.Print(indir + "/mergedUnfolded_{}.root".format(options.normalization))
    c.cd(1)
    ROOT.gPad.SetLogy()
    c.Print(indir + "/mergedUnfolded_{}_log.png".format(options.normalization))
    c.Print(indir + "/mergedUnfolded_{}_log.pdf".format(options.normalization))

    # rivet export
    from do import todoCatAll
    if len(todoCatAll) != 6:
        raise Exception(
            "Error: inconsistent number of categories in todoCatAll")
    rivet = ROOT.TFile("toRivet.root", "RECREATE")
    rivetNum = todoCatAll.index(options.variant) + 1
    if "area" == options.normalization:
        rivetNum += 10
    numAsStr = str(rivetNum)
    if len(numAsStr) == 1:
        numAsStr = "0" + numAsStr

    rivetName = "d" + numAsStr + "-x01-y01"
    print options.normalization, rivetNum, rivetName
    rivet.WriteTObject(result4Rivet, rivetName)
    rivet.Close()
    del rivet

    import os
    r2f = "/cvmfs/cms.cern.ch/slc6_amd64_gcc481/external/rivet/1.8.2-cms8/bin/root2flat"
    if not os.path.isfile(r2f):
        raise Exception("Cannot find root2flat. Rivet export failed")

    os.system(r2f + " toRivet.root")

    import yoda
    analysisobjects = yoda.readFLAT(rivetName + ".dat")
    #print type(analysisobjects)
    #print analysisobjects.keys()
    for k in analysisobjects:
        pth = "/CMS_2015_FWD071/" + rivetName
        #print dir(analysisobjects[k])
        #analysisobjects[k].setTitle(pth)
        #analysisobjects[k].setPath(pth)
        analysisobjects[k].setAnnotation("Title", pth)
        analysisobjects[k].setAnnotation("Path", pth)

    yoda.writeYODA(analysisobjects, rivetName + ".yoda")