def runExclusiveAnalysis(inFile, outFileName, runLumiList, mixFile):
    """event loop"""

    isData = True if 'Data' in inFile else False
    isSignal = True if 'MC13TeV_ppxz_' in inFile else False

    #bind main tree with pileup discrimination tree, if failed return
    tree = ROOT.TChain('analysis/data' if isSignal else 'tree')
    tree.AddFile(inFile)
    #try:
    #    pudiscr_tree=ROOT.TChain('pudiscr')
    #    baseName=os.path.basename(inFile)
    #    baseDir=os.path.dirname(inFile)
    #    pudiscr_file=os.path.join(baseDir,'pudiscr',baseName)
    #    if not os.path.isfile(pudiscr_file):
    #        raise ValueError(pudiscr_file+' does not exist')
    #    pudiscr_tree.AddFile(pudiscr_file)
    #    tree.AddFriend(pudiscr_tree)
    #    print 'Added pu tree for',inFile
    #except:
    #    #print 'Failed to add pu discrimination tree as friend for',inFile
    #    return

    #identify data-taking era
    era = None
    if isData:
        era = os.path.basename(inFile).split('_')[1]

    #check if it is signal and load
    signalPt = []
    mcEff = {}
    if isSignal:
        signalPt = [
            float(x) for x in re.findall(r'\d+', os.path.basename(inFile))[2:]
        ]
        for ch in ['eez', 'mmz', 'a']:
            effIn = ROOT.TFile.Open(
                '$CMSSW_BASE/src/TopLJets2015/TopAnalysis/plots/effsummary_%s_ptboson.root'
                % ch)
            pname = 'gen%srec_ptboson_ZH#rightarrowllbb_eff' % ch
            if ch == 'a': pname = 'genarec_ptboson_EWK #gammajj_eff'
            mcEff[ch] = effIn.Get(pname)
            effIn.Close()

    #filter events to mix according to tag if needed
    mixedRP = None
    xangleRelFracs = {}
    try:
        print 'Analysing mixfile', mixFile
        with open(mixFile, 'r') as cachefile:
            mixedRP = pickle.load(cachefile)

        #build the list of probabilities for the crossing angles in each era
        for key in mixedRP:
            mix_era, mix_xangle, mix_evcat = key
            if not mix_xangle in VALIDLHCXANGLES: continue
            n = len(mixedRP[key])
            xangleKey = (mix_era, mix_evcat)
            if not xangleKey in xangleRelFracs:
                xangleRelFracs[xangleKey] = ROOT.TH1F(
                    'xanglefrac_%s_%d' % xangleKey, '', len(VALIDLHCXANGLES),
                    0, len(VALIDLHCXANGLES))
            xbin = (mix_xangle - 120) / 10 + 1
            xangleRelFracs[xangleKey].SetBinContent(xbin, n)
        for xangleKey in xangleRelFracs:
            xangleRelFracs[xangleKey].Scale(
                1. / xangleRelFracs[xangleKey].Integral())
    except Exception as e:
        if mixFile: print e
        pass

    #start histograms
    ht = HistoTool()

    #main analysis histograms
    ht.add(
        ROOT.TH1F('mll', ';Dilepton invariant mass [GeV];Events', 50, 20, 250))
    ht.add(ROOT.TH1F('yll', ';Dilepton rapidity;Events', 50, 0, 5))
    ht.add(
        ROOT.TH1F('ptll', ';Dilepton transverse momentum [GeV];Events', 50, 0,
                  250))
    ht.add(ROOT.TH1F('l1eta', ';Lepton pseudo-rapidiy;Events', 50, 0, 2.5))
    ht.add(
        ROOT.TH1F('l1pt', ';Lepton transverse momentum [GeV];Events', 50, 0,
                  250))
    ht.add(ROOT.TH1F('l2eta', ';Lepton pseudo-rapidiy;Events', 50, 0, 2.5))
    ht.add(
        ROOT.TH1F('l2pt', ';Lepton transverse momentum [GeV];Events', 50, 0,
                  250))
    ht.add(ROOT.TH1F('acopl', ';A=1-|#Delta#phi|/#pi;Events', 50, 0, 1))
    ht.add(
        ROOT.TH1F('xangle', ';LHC crossing angle [#murad];Events', 4, 120,
                  160))
    ht.add(
        ROOT.TH1F('mpp', ';Di-proton invariant mass [GeV];Events', 50, 0,
                  3000))
    ht.add(ROOT.TH1F('ypp', ';Di-proton rapidity;Events', 50, 0, 2))
    ht.add(
        ROOT.TH2F(
            'mpp2d',
            ';Far di-proton invariant mass [GeV];Near di-proton invariant mass [GeV];Events',
            50, 0, 3000, 50, 0, 3000))
    ht.add(
        ROOT.TH2F('ypp2d',
                  ';Far di-proton rapidity;Near di-proton rapidity;Events', 50,
                  0, 2, 50, 0, 2))
    ht.add(ROOT.TH1F('mmass', ';Missing mass [GeV];Events', 50, 0, 3000))
    ht.add(ROOT.TH1F('ntk', ';Track multiplicity;Events', 5, 0, 5))
    ht.add(ROOT.TH1F('ppcount', ';pp candidates;Events', 3, 0, 3))
    ht.add(ROOT.TH1F('csi', ';#xi;Events', 50, 0, 0.3))
    ht.add(
        ROOT.TH2F('csi2d', ';#xi(far);#xi(near);Events', 50, 0, 0.3, 50, 0,
                  0.3))

    #pileup control
    ht.add(ROOT.TH1F('nvtx', ';Vertex multiplicity;Events', 50, 0, 100))
    ht.add(ROOT.TH1F('rho', ';Fastjet #rho;Events', 50, 0, 30))
    #ht.add(ROOT.TH1F('rfc',';Random forest classifier probability;Events',50,0,1))
    for d in ['HF', 'HE', 'EE', 'EB']:
        ht.add(
            ROOT.TH1F('PFMult' + d, ';PF multiplicity (%s);Events' % d, 50, 0,
                      1000))
        ht.add(
            ROOT.TH1F('PFHt' + d, ';PF HT (%s) [GeV];Events' % d, 50, 0, 1000))
        ht.add(
            ROOT.TH1F('PFPz' + d, ';PF P_{z} (%s) [TeV];Events' % d, 50, 0,
                      40))
    ht.add(
        ROOT.TH1F('met', ';Missing transverse energy [GeV];Events', 50, 0,
                  200))
    ht.add(ROOT.TH1F('metbits', ';MET filters;Events', 124, 0, 124))
    ht.add(ROOT.TH1F('njets', ';Jet multiplicity;Events', 5, 0, 5))
    ht.add(ROOT.TH1F('nch', ';Charged particle multiplicity;Events', 50, 0,
                     50))
    ht.add(ROOT.TH1F('nextramu', ';Additional muons ;Events', 10, 0, 10))
    ht.add(
        ROOT.TH1F('extramupt', ';Additional muon p_{T} [GeV] ;Events', 10, 0,
                  50))
    ht.add(
        ROOT.TH1F('extramueta', ';Additional muon pseudo-rapidty ;Events', 10,
                  0, 2.5))

    nEntries = tree.GetEntries()
    print '....analysing', nEntries, 'in', inFile, ', with output @', outFileName
    if mixedRP: print '    events mixed with', mixFile

    #loop over events
    rpData = {}
    selEvents = []
    summaryVars = 'cat:wgt:xangle:'
    summaryVars += 'l1pt:l1eta:l2pt:l2eta:acopl:bosonpt:'
    summaryVars += 'nch:nvtx:rho:PFMultSumHF:PFHtSumHF:PFPzSumHF:rfc:'
    summaryVars += 'csi1:csi2:mpp:mmiss:'
    summaryVars += 'mixcsi1:mixcsi2:mixmpp:mixmmiss:'
    summaryVars += 'mixemcsi1:mixemcsi2:mixemmpp:mixemmmiss'
    summaryVars = summaryVars.split(':')
    for i in xrange(0, nEntries):

        tree.GetEntry(i)

        #init "golden selected events for final statistcal analysis"
        goldenSel = None

        if i % 1000 == 0:
            sys.stdout.write('\r [ %d/100 ] done' %
                             (int(float(100. * i) / float(nEntries))))

        #base event selection
        if tree.evcat == 11 * 11 and not tree.isSS: evcat = 'ee'
        elif tree.evcat == EMU and not tree.isSS: evcat = 'em'
        elif tree.evcat == DIMUONS and not tree.isSS: evcat = 'mm'
        elif tree.evcat == 22:
            if isSignal:
                evcat = "a"
            else:
                if tree.hasATrigger: evcat = "a"
        elif tree.evcat == 0 and tree.hasZBTrigger: evcat == 'zbias'
        else: continue

        #assign data-taking era and crossing angle
        evEra = era
        beamXangle = tree.beamXangle
        if not isData:
            evEra = getRandomEra()
            if not isSignal:
                xangleKey = (evEra, DIMUONS)
                xbin = ROOT.TMath.FloorNint(
                    xangleRelFracs[xangleKey].GetRandom())
                beamXangle = VALIDLHCXANGLES[xbin]

        #data specific filters
        if isData:

            #reject invalid beam crossing angles
            if not beamXangle in VALIDLHCXANGLES: continue

            #check RPs are in
            if not isValidRunLumi(tree.run, tree.lumi, runLumiList): continue

        #lepton kinematics
        l1p4 = ROOT.TLorentzVector(0, 0, 0, 0)
        l2p4 = ROOT.TLorentzVector(0, 0, 0, 0)
        acopl = 0
        if tree.evcat != 22 and tree.evcat != 0:
            acopl = 1.0 - abs(ROOT.TVector2.Phi_mpi_pi(
                tree.l1phi - tree.l2phi)) / ROOT.TMath.Pi()
            l1p4.SetPtEtaPhiM(tree.l1pt, tree.l1eta, tree.l1phi, tree.ml1)
            l2p4.SetPtEtaPhiM(tree.l2pt, tree.l2eta, tree.l2phi, tree.ml2)
            if l1p4.Pt() < l2p4.Pt(): l1p4, l2p4 = l2p4, l1p4
            #if abs(l1p4.Eta())>2.1 : continue

        #boson kinematics
        boson = ROOT.TLorentzVector(0, 0, 0, 0)
        boson.SetPtEtaPhiM(tree.bosonpt, tree.bosoneta, tree.bosonphi,
                           tree.mboson)
        isZ = tree.isZ
        isA = tree.isA
        isHighPtZ = (boson.Pt() > 50)
        isLowPtZ = (boson.Pt() < 10)

        #PU-related variables
        #for signal most of these will be overriden by mixing
        n_extra_mu, nvtx, nch, rho, met, njets, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
        extra_muons = []
        if not isSignal:
            nvtx = tree.nvtx
            nch = tree.nchPV
            rho = tree.rho
            met = tree.met_pt
            njets = tree.nj
            PFMultSumHF = tree.PFMultSumHF
            PFHtSumHF = tree.PFHtSumHF
            PFPzSumHF = tree.PFPzSumHF
            #rfc=getattr(tree,'rfc_%d'%beamXangle)
            for im in range(tree.nrawmu):
                mup4 = ROOT.TLorentzVector(0, 0, 0, 0)
                mup4.SetPtEtaPhiM(tree.rawmu_pt[im], tree.rawmu_eta[im] / 10.,
                                  tree.rawmu_phi[im] / 10., 0.105)
                if mup4.DeltaR(l1p4) < 0.05: continue
                if mup4.DeltaR(l2p4) < 0.05: continue
                extra_muons.append(ROOT.TLorentzVector(mup4))
            n_extra_mu = len(extra_muons)

        #proton tracks (standard and mixed)
        far_rptks, near_rptks = None, None
        if isSignal or isData:
            far_rptks = getTracksPerRomanPot(tree)
            near_rptks = getTracksPerRomanPot(tree, False, False)

        #if data and there is nothing to mix store the main characteristics of the event and continue
        if isData and not mixedRP:

            #for Z->mm use only 10% otherwise we have way too many events to do this efficiently
            if (isZ and tree.evcat == DIMUONS and isLowPtZ
                    and random.random() < 0.1) or evcat == 'em':

                rpDataKey = (era, beamXangle, int(tree.evcat))
                if not rpDataKey in rpData: rpData[rpDataKey] = []
                rpData[rpDataKey].append(
                    MixedEvent(beamXangle, [
                        len(extra_muons), nvtx, rho, PFMultSumHF, PFHtSumHF,
                        PFPzSumHF, rfc
                    ], far_rptks, near_rptks))
            continue

        #do the mixing
        mixed_far_rptks, mixed_far_1rptk = {}, {}
        try:
            for mixEvCat in [DIMUONS, EMU]:
                mixedEvKey = (evEra, beamXangle, mixEvCat)
                mixedEv = random.choice(mixedRP[mixedEvKey])
                mixed_far_rptks[mixEvCat] = mixedEv.far_rptks
                if far_rptks and mixed_far_rptks[mixEvCat]:
                    if random.random() < 0.5:
                        mixed_far_1rptk[mixEvCat] = (
                            mixed_far_rptks[mixEvCat][0], far_rptks[1])
                    else:
                        mixed_far_1rptk[mixEvCat] = (
                            far_rptks[0], mixed_far_rptks[mixEvCat][1])

                #for signal add the tracks to the simulated ones
                #assign pileup characteristics from the Z->mm low pT events
                if isSignal:
                    if mixEvCat == DIMUONS:
                        n_extra_mu, nvtx, rho, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc = mixedEv.puDiscr
                    tksPos = mixed_far_rptks[mixEvCat][0] + far_rptks[0]
                    shuffle(tksPos)
                    tksNeg = mixed_far_rptks[mixEvCat][1] + far_rptks[1]
                    shuffle(tksNeg)
                    mixed_far_rptks[mixEvCat] = (tksPos, tksNeg)

        except Exception as e:
            print e
            print evEra, beamXangle, mixEvCat, '->', mixedEvKey
            pass

        #prepare the combinations of protons with central detector to fill histograms for
        mon_tasks = []
        if isData:
            mon_tasks.append((far_rptks, near_rptks, ''))
            if DIMUONS in mixed_far_1rptk:
                mon_tasks.append((mixed_far_1rptk[DIMUONS], None, '_mix1'))
                mon_tasks.append((mixed_far_rptks[DIMUONS], None, '_mix2'))
            if EMU in mixed_far_1rptk:
                mon_tasks.append((mixed_far_1rptk[EMU], None, '_mixem1'))
                mon_tasks.append((mixed_far_rptks[EMU], None, '_mixem2'))
        else:
            if DIMUONS in mixed_far_rptks:
                mon_tasks.append((mixed_far_rptks[DIMUONS], None, ''))
            if EMU in mixed_far_rptks:
                mon_tasks.append((mixed_far_rptks[EMU], None, '_mixem2'))
            if isSignal:
                mon_tasks.append((far_rptks, near_rptks, 'nopu'))

        #fill the histograms
        wgt = tree.evwgt
        hasAHighPurSelection = False
        for protons, near_protons, pfix in mon_tasks:

            #no calibration, not worth it...
            if not beamXangle in VALIDLHCXANGLES: continue

            #high purity selection for proton tracks
            highPur = True if protons and len(protons[0]) == 1 and len(
                protons[1]) == 1 else False
            if highPur:
                if protons[0][0] < 0.02 or protons[0][0] > 0.18:
                    highPur = False
                if protons[1][0] < 0.02 or protons[1][0] > 0.18:
                    highPur = False

            #check if Z and di-proton combination is consistent with elastic scattering
            pp = buildDiproton(protons)
            near_pp = buildDiproton(near_protons) if near_protons else None
            isElasticLike = False
            mmass = 0
            if pp:
                isElasticLike = (13000. - boson.E() - pp.E() > 0)
                inPP = ROOT.TLorentzVector(0, 0, 0, 13000.)
                if isElasticLike:
                    mmass = (pp - boson).M()

            #categories to fill
            cats = []
            cats.append(evcat)
            if isZ:
                cats.append(evcat + 'Z')
                if isHighPtZ: cats.append(evcat + 'hptZ')
                if isLowPtZ: cats.append(evcat + 'lptZ')
            if isElasticLike and highPur:
                ppCats = [c + 'hpur' for c in cats]
                cats += ppCats
            beamAngleCats = [c + '%d' % beamXangle for c in cats]
            cats += beamAngleCats

            #save he basic info on golden events
            saveGoldenSel = False
            if isSignal and pfix in ['', '_mixem2']:
                saveGoldenSel = True
            if isData and (isZ or isA):
                if pfix in ['', '_mix2', '_mixem2']:
                    saveGoldenSel = True

            if saveGoldenSel:
                if not goldenSel:
                    goldenSel = [
                        tree.evcat, wgt, beamXangle,
                        l1p4.Pt(),
                        l1p4.Eta(),
                        l2p4.Pt(),
                        l2p4.Eta(), acopl,
                        boson.Pt(), nch, nvtx, rho, PFMultSumHF, PFHtSumHF,
                        PFPzSumHF, rfc
                    ]
                task_protonInfo = [0, 0, 0, 0]
                if isElasticLike and highPur:
                    hasAHighPurSelection = True
                    task_protonInfo = [
                        protons[0][0], protons[1][0],
                        pp.M(), mmass
                    ]

                goldenSel += task_protonInfo

            #final plots (for signal correct wgt by efficiency curve and duplicate for mm channel)
            finalPlots = [[wgt, cats]]
            if isSignal:
                #signal has been pre-selected in the fiducial phase space so nEntries is
                #the effective number of events (or sum of weights) generated
                if isZ:
                    finalPlots = [
                        [wgt * mcEff['eez'].Eval(boson.Pt()) / nEntries, cats],
                        [
                            wgt * mcEff['mmz'].Eval(boson.Pt()) / nEntries,
                            [
                                c.replace(evcat, 'mm') for c in cats
                                if c[0:2] == 'ee'
                            ]
                        ]
                    ]
                else:
                    finalPlots = [[
                        wgt * mcEff['a'].Eval(boson.Pt()) / nEntries, cats
                    ]]

            for pwgt, pcats in finalPlots:

                #boson kinematics
                ht.fill((l1p4.Pt(), pwgt), 'l1pt', pcats, pfix)
                ht.fill((l2p4.Pt(), pwgt), 'l2pt', pcats, pfix)
                ht.fill((abs(l1p4.Eta()), pwgt), 'l1eta', pcats, pfix)
                ht.fill((abs(l2p4.Eta()), pwgt), 'l2eta', pcats, pfix)
                ht.fill((acopl, pwgt), 'acopl', pcats, pfix)
                ht.fill((boson.M(), pwgt), 'mll', pcats, pfix)
                ht.fill((abs(boson.Rapidity()), pwgt), 'yll', pcats, pfix)
                ht.fill((boson.Pt(), pwgt), 'ptll', pcats, pfix)

                #pileup related
                ht.fill((beamXangle, pwgt), 'xangle', pcats, pfix)
                ht.fill((nvtx, pwgt), 'nvtx', pcats, pfix)
                ht.fill((rho, pwgt), 'rho', pcats, pfix)
                ht.fill((met, pwgt), 'met', pcats, pfix)
                ht.fill((njets, pwgt), 'njets', pcats, pfix)
                ht.fill((nch, pwgt), 'nch', pcats, pfix)
                #ht.fill((getattr(tree,'rfc_%d'%beamXangle),pwgt), 'rfc',         pcats,pfix)
                ht.fill((PFMultSumHF, pwgt), 'PFMultHF', pcats, pfix)
                ht.fill((PFHtSumHF, pwgt), 'PFHtHF', pcats, pfix)
                ht.fill((PFPzSumHF / 1.e3, pwgt), 'PFPZHF', pcats, pfix)
                ht.fill((n_extra_mu, pwgt), 'nextramu', pcats, pfix)
                if not isSignal:
                    ht.fill((tree.metfilters, pwgt), 'metbits', pcats, pfix)
                    for sd in ['HE', 'EE', 'EB']:
                        ht.fill((getattr(tree, 'PFMultSum' + sd), pwgt),
                                'PFMult' + sd, pcats, pfix)
                        ht.fill((getattr(tree, 'PFHtSum' + sd), pwgt),
                                'PFHt' + sd, pcats, pfix)
                        ht.fill((getattr(tree, 'PFPzSum' + sd) / 1.e3, pwgt),
                                'PFPZ' + sd, pcats, pfix)
                    for mp4 in extra_muons:
                        ht.fill((mp4.Pt(), pwgt), 'extramupt', pcats, pfix)
                        ht.fill((abs(mp4.Eta()), pwgt), 'extramueta', pcats,
                                pfix)

                #proton counting and kinematics
                for irp, rpside in [(0, 'pos'), (1, 'neg')]:
                    ht.fill((len(protons[irp]), pwgt), 'ntk', pcats,
                            rpside + pfix)
                    for csi in protons[irp]:
                        ht.fill((csi, pwgt), 'csi', pcats, rpside + pfix)
                        if not near_protons: continue
                        if len(near_protons[irp]) == 0: continue
                        csi_near = near_protons[irp][0]
                        ht.fill((csi, csi_near, pwgt), 'csi2d', pcats,
                                rpside + pfix)

                #diproton kinematics
                if not pp:
                    ht.fill((0, pwgt), 'ppcount', pcats, pfix)
                    continue
                ht.fill((1, pwgt), 'ppcount', pcats, pfix)
                ht.fill((pp.M(), pwgt), 'mpp', pcats, pfix)
                ht.fill((abs(pp.Rapidity()), pwgt), 'ypp', pcats, pfix)
                if near_pp:
                    ht.fill((2, pwgt), 'ppcount', pcats, pfix)
                    ht.fill((pp.M(), near_pp.M(), pwgt), 'mpp2d', pcats, pfix)
                    ht.fill(
                        (abs(pp.Rapidity()), abs(near_pp.Rapidity()), pwgt),
                        'ypp2d', pcats, pfix)

                #the final variable
                ht.fill((mmass, pwgt), 'mmass', pcats, pfix)

        #select events
        if not hasAHighPurSelection: continue
        if not goldenSel: continue

        #fill missing variables with 0.'s (signal)
        nVarsMissed = len(summaryVars) - len(goldenSel)
        if nVarsMissed > 0: goldenSel += [0.] * nVarsMissed

        if isSignal:

            if isZ:
                #add a copy for ee
                eeGoldenSel = copy.copy(goldenSel)
                eeGoldenSel[0] = 11 * 11
                eeGoldenSel[1] = goldenSel[1] * mcEff['eez'].Eval(
                    boson.Pt()) / nEntries
                selEvents.append(eeGoldenSel)

                #add a copy for mm
                mmGoldenSel = copy.copy(goldenSel)
                mmGoldenSel[0] = DIMUONS
                mmGoldenSel[1] = goldenSel[1] * mcEff['mmz'].Eval(
                    boson.Pt()) / nEntries
                selEvents.append(mmGoldenSel)

            if isA:

                #add a copy for the photon
                aGoldenSel = copy.copy(goldenSel)
                aGoldenSel[0] = 22
                aGoldenSel[1] = goldenSel[1] * mcEff['a'].Eval(
                    boson.Pt()) / nEntries
                selEvents.append(aGoldenSel)

        else:

            selEvents.append(goldenSel)

    #dump events for the mixing
    nSelRPData = sum([len(rpData[x]) for x in rpData])
    if nSelRPData > 0:
        rpDataOut = outFileName.replace('.root', '.pck')
        print 'Saving', nSelRPData, 'events for mixing in', rpDataOut
        with open(rpDataOut, 'w') as cachefile:
            pickle.dump(rpData, cachefile, pickle.HIGHEST_PROTOCOL)

    #if there was no mixing don't do anything else
    if not mixFile: return

    #save results
    ht.writeToFile(outFileName)

    #dump events for fitting
    nSelEvents = len(selEvents)
    if nSelEvents > 0:
        print 'Adding', nSelEvents, 'selected events to', outFileName
        fOut = ROOT.TFile.Open(outFileName, 'UPDATE')
        fOut.cd()
        t = ROOT.TNtuple('data', 'data', ':'.join(summaryVars))
        for v in selEvents:
            t.Fill(array.array("f", v))
        t.Write()
        fOut.Close()
Exemple #2
0
def runExclusiveAnalysis(inFile, outFileName, runLumiList, mixFile):
    """event loop"""

    #identify data-taking era
    era = None
    isData = True if 'Data' in inFile else False
    if isData:
        era = os.path.basename(inFile).split('_')[1]

    #check if it is signal and load
    isSignal = True if 'MC13TeV_2017_PPZX_' in inFile else False
    signalPt = []
    mcEff = {}
    if isSignal:
        signalPt = [
            float(x) for x in re.findall(r'\d+', os.path.basename(inFile))[2:]
        ]
        for ch in ['ee', 'mm']:
            effIn = ROOT.TFile.Open(
                '$CMSSW_BASE/src/TopLJets2015/TopAnalysis/plots/effsummary_%sz_ptboson.root'
                % ch)
            mcEff[ch] = effIn.Get('gen%sz2trec_ptboson_ZH#rightarrowllbb_eff' %
                                  ch)
            effIn.Close()

    #filter events to mix according to tag if needed
    mixedRP = None
    try:
        with open(mixFile, 'r') as cachefile:
            mixedRP = pickle.load(cachefile)
    except:
        pass

    #start histograms
    ht = HistoTool()
    ht.add(ROOT.TH1F('nvtx', ';Vertex multiplicity;Events', 50, 0, 100))
    ht.add(ROOT.TH1F('rho', ';Fastjet #rho;Events', 50, 0, 30))
    ht.add(
        ROOT.TH1F('met', ';Missing transverse energy [GeV];Events', 50, 0,
                  200))
    ht.add(ROOT.TH1F('metbits', ';MET filters;Events', 124, 0, 124))
    ht.add(ROOT.TH1F('njets', ';Jet multiplicity;Events', 5, 0, 5))
    ht.add(ROOT.TH1F('nch', ';Charged particle multiplicity;Events', 50, 0,
                     50))
    ht.add(ROOT.TH1F('acopl', ';A=1-|#Delta#phi|/#pi;Events', 50, 0, 1))
    ht.add(ROOT.TH1F('l1eta', ';Lepton pseudo-rapidiy;Events', 50, 0, 2.5))
    ht.add(
        ROOT.TH1F('l1pt', ';Lepton transverse momentum [GeV];Events', 50, 0,
                  250))
    ht.add(ROOT.TH1F('l2eta', ';Lepton pseudo-rapidiy;Events', 50, 0, 2.5))
    ht.add(
        ROOT.TH1F('l2pt', ';Lepton transverse momentum [GeV];Events', 50, 0,
                  250))
    ht.add(
        ROOT.TH1F('mll', ';Dilepton invariant mass [GeV];Events', 50, 20, 250))
    ht.add(ROOT.TH1F('yll', ';Dilepton rapidity;Events', 50, 0, 5))
    ht.add(
        ROOT.TH1F('ptll', ';Dilepton transverse momentum [GeV];Events', 50, 0,
                  250))
    ht.add(ROOT.TH1F('minenfwd', ';min(E_{+},E_{-}) [GeV];Events', 20, 0, 300))
    ht.add(ROOT.TH1F('maxenfwd', ';max(E_{+},E_{-}) [GeV];Events', 20, 0, 300))
    ht.add(ROOT.TH1F('deltaenfwd', ';|E_{+}-E_{-}| [GeV];Events', 20, 0, 300))
    ht.add(ROOT.TH1F('sgny', ';y x sgn(LRG) ;Events', 20, -3, 3))
    ht.add(ROOT.TH1F('nextramu', ';Additional muons ;Events', 10, 0, 10))
    ht.add(
        ROOT.TH1F('extramupt', ';Additional muon p_{T} [GeV] ;Events', 10, 0,
                  50))
    ht.add(
        ROOT.TH1F('extramueta', ';Additional muon pseudo-rapidty ;Events', 10,
                  0, 2.5))
    ht.add(
        ROOT.TH1F('xangle', ';LHC crossing angle [#murad];Events', 4, 120,
                  160))
    ht.add(
        ROOT.TH1F('mpp', ';Di-proton invariant mass [GeV];Events', 50, 0,
                  3000))
    ht.add(ROOT.TH1F('ypp', ';Di-proton rapidity;Events', 50, 0, 2))
    ht.add(ROOT.TH1F('mmass', ';Missing mass [GeV];Events', 50, 0, 3000))
    ht.add(ROOT.TH1F('ntk', ';Track multiplicity;Events', 5, 0, 5))
    ht.add(ROOT.TH1F('csi', ';#xi;Events', 50, 0, 0.3))

    #start analysis
    tree = ROOT.TChain('analysis/data' if isSignal else 'tree')
    tree.AddFile(inFile)
    nEntries = tree.GetEntries()
    print '....analysing', nEntries, 'in', inFile, ', with output @', outFileName
    if mixedRP: print '    events mixed with', mixFile

    #loop over events
    rpData = {era: []}
    selEvents = []
    summaryVars = 'cat:wgt:nvtx:nch:xangle:l1pt:l1eta:l2pt:l2eta:acopl:bosonpt:mpp:mmiss:mpp2:mmiss2'.split(
        ':')
    for i in xrange(0, nEntries):

        tree.GetEntry(i)

        if i % 1000 == 0:
            sys.stdout.write('\r [ %d/100 ] done' %
                             (int(float(100. * i) / float(nEntries))))

        #base event selection
        if tree.evcat == 11 * 11: evcat = 'ee'
        elif tree.evcat == 11 * 13: evcat = 'em'
        elif tree.evcat == 13 * 13: evcat = 'mm'
        else: continue
        if tree.isSS: continue
        if isData and not isValidRunLumi(tree.run, tree.lumi, runLumiList):
            continue

        wgt = tree.evwgt
        nvtx = tree.nvtx
        nch = tree.nch
        rho = tree.rho
        met = tree.met_pt
        njets = 0 if isSignal else tree.nj

        #acoplanarity
        acopl = 1.0 - abs(ROOT.TVector2.Phi_mpi_pi(
            tree.l1phi - tree.l2phi)) / ROOT.TMath.Pi()

        l1p4 = ROOT.TLorentzVector(0, 0, 0, 0)
        l1p4.SetPtEtaPhiM(tree.l1pt, tree.l1eta, tree.l1phi, tree.ml1)
        l2p4 = ROOT.TLorentzVector(0, 0, 0, 0)
        l2p4.SetPtEtaPhiM(tree.l2pt, tree.l2eta, tree.l2phi, tree.ml2)
        if l1p4.Pt() < l2p4.Pt(): l1p4, l2p4 = l2p4, l1p4
        if abs(l1p4.Eta()) > 2.1: continue

        #boson kinematics
        boson = ROOT.TLorentzVector(0, 0, 0, 0)
        boson.SetPtEtaPhiM(tree.bosonpt, tree.bosoneta, tree.bosonphi,
                           tree.mboson)
        isZ = tree.isZ
        isHighPt = (boson.Pt() > 50)

        #possible diffractive-sensitive variabes
        en_posRG = tree.jsumposhfen
        en_negRG = tree.jsumneghfen
        extra_muons = []
        for im in range(tree.nrawmu):
            mup4 = ROOT.TLorentzVector(0, 0, 0, 0)
            mup4.SetPtEtaPhiM(tree.rawmu_pt[im], tree.rawmu_eta[im] / 10.,
                              tree.rawmu_phi[im] / 10., 0.105)
            if mup4.DeltaR(l1p4) < 0.05: continue
            if mup4.DeltaR(l2p4) < 0.05: continue
            extra_muons.append(ROOT.TLorentzVector(mup4))

        #proton tracks (standard and mixed)
        rptks, near_rptks = None, None
        beamXangle = tree.beamXangle
        if isSignal:
            beamXangle = signalPt[0]
            rptks = getTracksPerRomanPot(tree)
            far_rptks = getTracksPerRomanPot(tree, False, False)
            rptks = ([x / 0.0964
                      for x in rptks[0]], [x / 0.06159 for x in rptks[1]])
            far_rptks = ([x / 0.0964 for x in far_rptks[0]],
                         [x / 0.06159 for x in far_rptks[1]])
        if isData:
            rptks = getTracksPerRomanPot(tree)
            far_rptks = getTracksPerRomanPot(tree, False, False)

        mixed_beamXangle = None
        mixed_rptks = None
        mixed_1rptk = None
        if not isData:
            evEra = getRandomEra()
        else:
            evEra = era
            if not mixedRP:
                if evcat == 'em' and tree.bosonpt > 50:
                    rpData[era].append(
                        MixedEvent(beamXangle, nvtx, rho, rptks, far_rptks,
                                   extra_muons, en_posRG, en_negRG))
                continue
        try:
            mixedEv = random.choice(mixedRP[evEra])
            mixed_beamXangle = mixedEv.beamXangle
            mixed_rptks = mixedEv.far_rptks
            if rptks and mixed_rptks:
                if random.random() < 0.5:
                    mixed_1rptk = (mixed_rptks[0], rptks[1])
                else:
                    mixed_1rptk = (rptks[0], mixed_rptks[1])
        except:
            pass

        #fill histograms
        goldenSel = None
        mon_tasks = []
        if isData:
            mon_tasks.append((rptks, beamXangle, ''))
            mon_tasks.append((mixed_1rptk, mixed_beamXangle, '_mix1'))
            mon_tasks.append((mixed_rptks, mixed_beamXangle, '_mix2'))
        else:
            if isSignal:
                #embed pileup to signal
                tksPos = mixed_rptks[0] + rptks[0]
                shuffle(tksPos)
                tksNeg = mixed_rptks[1] + rptks[1]
                shuffle(tksNeg)
                mixed_rptks = (tksPos, tksNeg)
                mixed_beamXangle = beamXangle
                mon_tasks.append((rptks, beamXangle, 'nopu'))
            mon_tasks.append((mixed_rptks, mixed_beamXangle, ''))

        for protons, xangle, pfix in mon_tasks:

            #no calibration, not worth it...
            if not xangle in VALIDLHCXANGLES: continue

            #high purity selection for proton tracks
            highPur = True if protons and len(protons[0]) == 1 and len(
                protons[1]) == 1 else False
            if highPur:
                if protons[0][0] < 0.05 or protons[0][0] > 0.20:
                    highPur = False
                if protons[1][0] < 0.05 or protons[1][0] > 0.20:
                    highPur = False

            #the famous cut
            noExtraMu = True
            if len(extra_muons) > 1: noExtraMu = False

            #check if Z and di-proton combination is consistent with elastic scattering
            pp = buildDiproton(protons)
            isElasticLike = False
            mmass = 0
            if pp:
                isElasticLike = (13000. - boson.E() - pp.E() > 0)
                inPP = ROOT.TLorentzVector(0, 0, 0, 13000.)
                if isElasticLike:
                    mmass = (pp - boson).M()

            #categories to fill
            cats = []
            cats.append(evcat)
            if isZ: cats.append(evcat + 'Z')
            if isHighPt: cats.append(evcat + 'hpt')
            if isZ and isHighPt: cats.append(evcat + 'hptZ')
            if isElasticLike:
                ppCats = [c + 'elpp' for c in cats]
                if highPur:
                    ppCats += [c + 'elpphighPur' for c in cats]
                if isZ:
                    cats += ppCats + [c + '%d' % xangle for c in ppCats]
            if noExtraMu:
                extrmucats = [c + 'noextramu' for c in cats]
                cats += extrmucats
            if nvtx < 2 and len(protons[0]) + len(protons[1]) == 1:
                if (en_negRG == 0 and en_posRG > 0) or (en_negRG > 0
                                                        and en_posRG == 0):
                    diffCats = [c + 'diff' for c in cats]
                    cats += diffCats

            if len(pfix) != 0:
                cats = [c for c in cats if 'elpp' in c]

            if (isData and 'mix' in pfix) or (isSignal and pfix == ''):
                if isZ and isElasticLike and highPur:
                    if goldenSel:
                        goldenSel += [pp.M(), mmass]
                    else:
                        goldenSel = [
                            tree.evcat, wgt, nvtx, nch, xangle,
                            l1p4.Pt(),
                            l1p4.Eta(),
                            l2p4.Pt(),
                            l2p4.Eta(), acopl,
                            boson.Pt(),
                            pp.M(), mmass
                        ]

            #final plots (for signal correct wgt by efficiency curve and duplicate for mm channel)
            finalPlots = [[wgt, cats]]
            if isSignal:
                finalPlots = [
                    [wgt * mcEff['ee'].Eval(boson.Pt()) / nEntries, cats],
                    [
                        wgt * mcEff['mm'].Eval(boson.Pt()) / nEntries,
                        [
                            c.replace(evcat, 'mm') for c in cats
                            if c[0:2] == 'ee'
                        ]
                    ]
                ]

            for pwgt, pcats in finalPlots:

                ht.fill((nvtx, pwgt), 'nvtx', pcats, pfix)
                ht.fill((rho, pwgt), 'rho', pcats, pfix)
                ht.fill((met, pwgt), 'met', pcats, pfix)
                ht.fill((tree.metfilters, pwgt), 'metbits', pcats, pfix)
                ht.fill((njets, pwgt), 'njets', pcats, pfix)
                ht.fill((nch, pwgt), 'nch', pcats, pfix)
                ht.fill((l1p4.Pt(), pwgt), 'l1pt', pcats, pfix)
                ht.fill((l2p4.Pt(), pwgt), 'l2pt', pcats, pfix)
                ht.fill((abs(l1p4.Eta()), pwgt), 'l1eta', pcats, pfix)
                ht.fill((abs(l2p4.Eta()), pwgt), 'l2eta', pcats, pfix)
                ht.fill((acopl, pwgt), 'acopl', pcats, pfix)
                ht.fill((boson.M(), pwgt), 'mll', pcats, pfix)
                ht.fill((abs(boson.Rapidity()), pwgt), 'yll', pcats, pfix)
                ht.fill((boson.Pt(), pwgt), 'ptll', pcats, pfix)
                ht.fill((xangle, pwgt), 'xangle', pcats, pfix)
                ht.fill((min(en_posRG, en_negRG), pwgt), 'minenfwd', pcats,
                        pfix)
                ht.fill((max(en_posRG, en_negRG), pwgt), 'maxenfwd', pcats,
                        pfix)
                ht.fill((abs(en_posRG - en_negRG), pwgt), 'deltaenfwd', pcats,
                        pfix)
                sgnY = boson.Rapidity(
                ) if en_posRG < en_negRG else -boson.Rapidity()
                ht.fill((sgnY, pwgt), 'sgny', pcats, pfix)
                ht.fill((len(extra_muons), pwgt), 'nextramu', pcats, pfix)
                for mp4 in extra_muons:
                    ht.fill((mp4.Pt(), pwgt), 'extramupt', pcats, pfix)
                    ht.fill((abs(mp4.Eta()), pwgt), 'extramueta', pcats, pfix)

                for irp, rpside in [(0, 'pos'), (1, 'neg')]:
                    ht.fill((len(protons[irp]), pwgt), 'ntk', pcats,
                            rpside + pfix)
                if not pp: continue
                ht.fill((pp.M(), pwgt), 'mpp', pcats, pfix)
                ht.fill((abs(pp.Rapidity()), pwgt), 'ypp', pcats, pfix)
                ht.fill((mmass, pwgt), 'mmass', pcats, pfix)
                for irp, rpside in [(0, 'pos'), (1, 'neg')]:
                    ht.fill((len(protons[irp]), pwgt), 'ntk', pcats,
                            rpside + pfix)
                    for csi in protons[irp]:
                        ht.fill((csi, pwgt), 'csi', pcats, rpside + pfix)

        #select events
        if goldenSel:
            nVarsMissed = len(summaryVars) - len(goldenSel)
            if nVarsMissed > 0: goldenSel += [0.] * nVarsMissed

            if isSignal:

                #add a copy for ee
                eeGoldenSel = copy.copy(goldenSel)
                eeGoldenSel[0] = 11 * 11
                eeGoldenSel[1] = goldenSel[1] * mcEff['ee'].Eval(
                    boson.Pt()) / nEntries
                selEvents.append(eeGoldenSel)

                #add a copy for mm
                mmGoldenSel = copy.copy(goldenSel)
                mmGoldenSel[0] = 13 * 13
                mmGoldenSel[1] = goldenSel[1] * mcEff['mm'].Eval(
                    boson.Pt()) / nEntries
                selEvents.append(mmGoldenSel)

            else:

                selEvents.append(goldenSel)

    #dump events for the mixing
    nSelRPData = len(rpData[era])
    if nSelRPData:
        rpDataOut = outFileName.replace('.root', '.pck')
        print 'Saving', nSelRPData, 'events for mixing in', rpDataOut
        with open(rpDataOut, 'w') as cachefile:
            pickle.dump(rpData, cachefile, pickle.HIGHEST_PROTOCOL)

    #save results
    ht.writeToFile(outFileName)

    #dump events for fitting
    nSelEvents = len(selEvents)
    if nSelEvents > 0:
        print 'Adding', nSelEvents, 'selected events to', outFileName
        fOut = ROOT.TFile.Open(outFileName, 'UPDATE')
        fOut.cd()
        t = ROOT.TNtuple('data', 'data', ':'.join(summaryVars))
        for v in selEvents:
            t.Fill(array.array("f", v))
        t.Write()
        fOut.Close()
def runExclusiveAnalysis(inFile,
                         outFileName,
                         runLumiList,
                         effDir,
                         ppsEffFile,
                         maxEvents=-1,
                         sighyp=0,
                         mixDir=None):
    """event loop"""

    global MIXEDRPSIG
    global ALLOWPIXMULT

    isData = True if 'Data' in inFile else False
    era = os.path.basename(inFile).split('_')[1] if isData else None
    isDY = isDYFile(inFile)
    isSignal, isPreTS2Signal = isSignalFile(inFile)
    isFullSimSignal = True if isSignal and 'fullsim' in inFile else False
    isPhotonSignal = isPhotonSignalFile(inFile)
    gen_mX = signalMassPoint(inFile) if isSignal else 0.

    #open this just once as it may be quite heavy in case it's not data or signal
    if mixDir:

        print 'Collecting events from the mixing bank'
        mixFiles = [f for f in os.listdir(mixDir) if '.pck' in f]

        #open just the necessary for signal and data
        if isSignal or isData:
            allowedEras = ['2017%s' % x for x in 'BCDEF']
            if isSignal:
                allowedAngle = re.search('xangle_(\d+)', inFile).group(1)
                mixFiles = [f for f in mixFiles if allowedAngle in f]
                if isPreTS2Signal:
                    allowedEras = ['2017%s' % x for x in 'BCD']
                else:
                    allowedEras = ['2017%s' % x for x in 'DEF']
            if isData:
                allowedEras = [era]
            mixFiles = [f for f in mixFiles if f.split('_')[1] in allowedEras]

        MIXEDRP = defaultdict(list)
        for f in mixFiles:
            print '\t', f
            with open(os.path.join(mixDir, f), 'r') as cachefile:
                rpData = pickle.load(cachefile)
                for key in rpData:
                    MIXEDRP[key] += rpData[key]
        print '\t size of mixing bank is', sys.getsizeof(MIXEDRP), 'byte'

    #bind main tree with pileup discrimination tree, if failed return
    tree = ROOT.TChain(
        'analysis/data' if isSignal and not isFullSimSignal else 'tree')
    tree.AddFile(inFile)
    #try:
    #    pudiscr_tree=ROOT.TChain('pudiscr')
    #    baseName=os.path.basename(inFile)
    #    baseDir=os.path.dirname(inFile)
    #    pudiscr_file=os.path.join(baseDir,'pudiscr',baseName)
    #    if not os.path.isfile(pudiscr_file):
    #        raise ValueError(pudiscr_file+' does not exist')
    #    pudiscr_tree.AddFile(pudiscr_file)
    #    tree.AddFriend(pudiscr_tree)
    #    print 'Added pu tree for',inFile
    #except:
    #    #print 'Failed to add pu discrimination tree as friend for',inFile
    #    return

    #check if it is signal and load
    signalPt = []
    ppsEffReader = None
    mcEff = {}
    if isSignal:
        ppsEffReader = PPSEfficiencyReader(ppsEffFile)
        signalPt = [
            float(x) for x in re.findall(r'\d+', os.path.basename(inFile))[2:]
        ]
        for ch in ['eez', 'mmz', 'a']:
            effIn = ROOT.TFile.Open('%s/effsummary_%s_ptboson.root' %
                                    (effDir, ch))
            pname = 'gen%srec_ptboson_ZH#rightarrowllbb_eff' % ch
            if ch == 'a': pname = 'genarec_ptboson_EWK #gammajj_eff'
            mcEff[ch] = effIn.Get(pname)
        effIn.Close()

    #start event mixing tool
    print MIXEDRP.keys()
    evMixTool = EventMixingTool(mixedRP=MIXEDRP, validAngles=VALIDLHCXANGLES)
    print 'Allowed pixel multiplicity is', ALLOWPIXMULT

    #start histograms
    ht = HistoTool()

    if isSignal:
        ht.add(
            ROOT.TH2F('sighyp', ';Initial category; Final category;Events', 16,
                      0, 16, 16, 0, 16))
        for i in range(16):
            lab = "|{0:04b}>".format(i)
            ht.histos['sighyp']['inc'].GetXaxis().SetBinLabel(i + 1, lab)
            ht.histos['sighyp']['inc'].GetYaxis().SetBinLabel(i + 1, lab)
    ht.add(ROOT.TH1F('catcount', ';Proton selection category;Events', 6, 0, 6))
    for i, c in enumerate(['inc', '=2s', 'mm', 'ms', 'sm', 'ss']):
        ht.histos['catcount']['inc'].GetXaxis().SetBinLabel(i + 1, c)

    #main analysis histograms
    ht.add(ROOT.TH1F('nvtx', ';Vertex multiplicity;Events', 50, 0, 100))
    ht.add(ROOT.TH1F('rho', ';Fastjet #rho;Events', 50, 0, 50))
    ht.add(
        ROOT.TH1F('xangle', ';LHC crossing angle [#murad];Events', 4, 120,
                  160))
    ht.add(ROOT.TH1F('mll', ';Invariant mass [GeV];Events', 50, 76, 106))
    ht.add(ROOT.TH1F('mll_full', ';Invariant mass [GeV];Events', 50, 0, 250))
    ht.add(ROOT.TH1F('yll', ';Rapidity;Events', 50, -3, 3))
    ht.add(ROOT.TH1F('etall', ';Pseudo-rapidity;Events', 50, -6, 6))
    ht.add(ROOT.TH1F('ptll', ';Transverse momentum [GeV];Events', 50, 0, 250))
    ht.add(
        ROOT.TH1F('ptll_high', ';Transverse momentum [GeV];Events', 50, 50,
                  500))
    ht.add(ROOT.TH1F('l1eta', ';Pseudo-rapidiy;Events', 50, 0, 2.5))
    ht.add(ROOT.TH1F('l1pt', ';Transverse momentum [GeV];Events', 50, 0, 250))
    ht.add(ROOT.TH1F('l2eta', ';Pseudo-rapidiy;Events', 50, 0, 2.5))
    ht.add(ROOT.TH1F('l2pt', ';Transverse momentum [GeV];Events', 50, 0, 250))
    ht.add(ROOT.TH1F('acopl', ';A=1-|#Delta#phi|/#pi;Events', 50, 0, 1))
    ht.add(ROOT.TH1F('costhetacs', ';cos#theta_{CS};Events', 50, -1, 1))

    #pileup control
    #ht.add(ROOT.TH1F('rfc',';Random forest classifier probability;Events',50,0,1))
    for d in ['HF', 'HE', 'EE', 'EB']:
        ht.add(
            ROOT.TH1F('PFMult' + d, ';PF multiplicity (%s);Events' % d, 50, 0,
                      1000))
        ht.add(
            ROOT.TH1F('PFHt' + d, ';PF HT (%s) [GeV];Events' % d, 50, 0, 1000))
        ht.add(
            ROOT.TH1F('PFPz' + d, ';PF P_{z} (%s) [TeV];Events' % d, 50, 0,
                      40))
    ht.add(
        ROOT.TH1F('met', ';Missing transverse energy [GeV];Events', 50, 0,
                  200))
    ht.add(ROOT.TH1F('mpf', ';MPF;Events', 50, -5, 5))
    ht.add(ROOT.TH1F('metbits', ';MET filters;Events', 124, 0, 124))
    ht.add(ROOT.TH1F('njets', ';Jet multiplicity;Events', 5, 0, 5))
    ht.add(ROOT.TH1F('zjb', ';Z-jet balance [GeV];Events', 50, -150, 150))
    ht.add(ROOT.TH1F('zj2b', ';Z-2 jets balance [GeV];Events', 50, -150, 150))
    ht.add(ROOT.TH1F('nch', ';Charged particle multiplicity;Events', 50, 0,
                     50))
    ht.add(ROOT.TH1F('nextramu', ';Additional muons ;Events', 10, 0, 10))
    ht.add(
        ROOT.TH1F('extramupt', ';Additional muon p_{T} [GeV] ;Events', 10, 0,
                  50))
    ht.add(
        ROOT.TH1F('extramueta', ';Additional muon pseudo-rapidty ;Events', 10,
                  0, 2.5))

    #RP control
    ht.add(
        ROOT.TH1F('mpp', ';Di-proton invariant mass [GeV];Events', 50, 0,
                  3000))
    ht.add(ROOT.TH1F('pzpp', ';Di-proton p_{z} [GeV];Events', 50, -750, 750))
    ht.add(ROOT.TH1F('ypp', ';Di-proton rapidity;Events', 50, -2.5, 2.5))
    ht.add(
        ROOT.TH1F('mmass_full', ';Missing mass [GeV];Events', 50, -1000, 3000))
    ht.add(ROOT.TH1F('mmass', ';Missing mass [GeV];Events', 50, 0, 3000))
    ht.add(ROOT.TH1F('ntk', ';Track multiplicity;Events', 5, 0, 5))
    ht.add(ROOT.TH1F('ppcount', ';pp candidates;Events', 3, 0, 3))
    ht.add(ROOT.TH1F('csi', ';#xi;Events', 50, 0, 0.3))

    nEntries = tree.GetEntries()
    print '....analysing', nEntries, 'in', inFile, ', with output @', outFileName
    if maxEvents > 0:
        nEntries = min(maxEvents, nEntries)
        print '      will process', nEntries, 'events'

    #compute number of events weighted by target pz spectrum
    nSignalWgtSum = 0.
    if isSignal:
        print 'Checking how many events are in the fiducial RP area...'
        for i in xrange(0, nEntries):
            tree.GetEntry(i)
            nSignalWgtSum += ROOT.TMath.Gaus(tree.gen_pzpp, 0,
                                             0.391 * gen_mX + 624)
        print '...signal weight sum set to', nSignalWgtSum, ' from ', nEntries, 'raw events'

    #start output and tree
    fOut = ROOT.TFile.Open(outFileName, 'RECREATE')
    evSummary = EventSummary()
    tOut = ROOT.TTree('data', 'data')
    evSummary.attachToTree(tOut)

    #summary events for the mixing
    rpData = {}

    #loop over events
    nfail = [0, 0, 0]
    for i in xrange(0, nEntries):

        tree.GetEntry(i)

        if i % 500 == 0:
            drawProgressBar(float(i) / float(nEntries))

        if isSignal:
            if isPhotonSignal and tree.evcat != SINGLEPHOTON: continue
            if not isPhotonSignal and tree.evcat == SINGLEPHOTON: continue

        isOffZ = False
        if tree.mboson > 101:
            if tree.evcat == DIELECTRONS or tree.evcat == DIMUONS:
                isOffZ = True

        #base event selection
        if tree.evcat == DIELECTRONS and tree.isZ:
            evcat = 'ee'
        elif tree.evcat == EMU and not tree.isSS:
            evcat = 'em'
        elif tree.evcat == DIMUONS and tree.isZ:
            evcat = 'mm'
        elif isOffZ:
            evcat = 'offz'
        elif tree.evcat == SINGLEPHOTON and (isPhotonSignal
                                             or tree.hasATrigger):
            evcat = "a"
        elif tree.evcat == 0 and tree.hasZBTrigger:
            evcat == 'zbias'
        else:
            nfail[0] += 1
            continue

        #assign data-taking era and crossing angle
        evEra = era
        beamXangle = tree.beamXangle
        if not isData:
            evEra = getRandomEra(isSignal, isPreTS2Signal)
            if not isSignal:
                xbin = evMixTool.getRandomLHCCrossingAngle(
                    evEra=evEra, evCat=SINGLEPHOTON if tree.isA else DIMUONS)
                beamXangle = VALIDLHCXANGLES[xbin]

        #check if RP is in (MC assume true by default)
        isRPIn = False if isData else True
        if isData and beamXangle in VALIDLHCXANGLES and isValidRunLumi(
                tree.run, tree.lumi, runLumiList):
            isRPIn = True
        if not isRPIn: nfail[1] += 1

        #lepton kinematics
        l1p4 = ROOT.TLorentzVector(0, 0, 0, 0)
        l2p4 = ROOT.TLorentzVector(0, 0, 0, 0)
        costhetacs = 0
        acopl = 0
        if tree.evcat != SINGLEPHOTON and tree.evcat != 0:
            acopl = 1.0 - abs(ROOT.TVector2.Phi_mpi_pi(
                tree.l1phi - tree.l2phi)) / ROOT.TMath.Pi()
            l1p4.SetPtEtaPhiM(tree.l1pt, tree.l1eta, tree.l1phi, tree.ml1)
            l2p4.SetPtEtaPhiM(tree.l2pt, tree.l2eta, tree.l2phi, tree.ml2)
            costhetacs = computeCosThetaStar(l1p4, l2p4)

            #force ordering by pT (before they were ordered by charge to compute costhetacs)
            if l1p4.Pt() < l2p4.Pt():
                l1p4, l2p4 = l2p4, l1p4

        #boson kinematics
        boson = ROOT.TLorentzVector(0, 0, 0, 0)
        boson.SetPtEtaPhiM(tree.bosonpt, tree.bosoneta, tree.bosonphi,
                           tree.mboson)
        isZ = tree.isZ
        isA = tree.isA

        #for the signal-electron hypothesis this cut needs to be applied
        hasEEEBTransition = False
        if isZ:
            if abs(l1p4.Eta()) > 1.4442 and abs(l1p4.Eta()) < 1.5660:
                hasEEEBTransition = True
            if abs(l2p4.Eta()) > 1.4442 and abs(l2p4.Eta()) < 1.5660:
                hasEEEBTransition = True

        #PU-related variables
        #for signal most of these will be overriden by mixing
        n_extra_mu, nvtx, nch, rho, met, njets, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
        mpf, zjb, zj2b = 0, 0, 0
        extra_muons = []
        if isFullSimSignal or not isSignal:
            nvtx = tree.nvtx
            nch = tree.nchPV
            rho = tree.rho
            met = tree.met_pt
            metphi = tree.met_phi
            njets = tree.nj
            PFMultSumHF = tree.PFMultSumHF
            PFHtSumHF = tree.PFHtSumHF
            PFPzSumHF = tree.PFPzSumHF
            #rfc=getattr(tree,'rfc_%d'%beamXangle)
            for im in range(tree.nrawmu):
                mup4 = ROOT.TLorentzVector(0, 0, 0, 0)
                mup4.SetPtEtaPhiM(tree.rawmu_pt[im], tree.rawmu_eta[im] / 10.,
                                  tree.rawmu_phi[im] / 10., 0.105)
                if mup4.DeltaR(l1p4) < 0.05: continue
                if mup4.DeltaR(l2p4) < 0.05: continue
                extra_muons.append(ROOT.TLorentzVector(mup4))
            n_extra_mu = len(extra_muons)

            metp4 = ROOT.TLorentzVector(0, 0, 0, 0)
            metp4.SetPtEtaPhiM(met, 0, metphi, 0)
            mpf = 1. + (metp4.Px() * boson.Px() +
                        metp4.Py() * boson.Py()) / (boson.Pt()**2 + 1.0e-6)
            if njets > 0:
                zjb = tree.j1pt - boson.Pt()
                if njets > 1:
                    j1p4 = ROOT.TLorentzVector(0, 0, 0, 0)
                    j1p4.SetPtEtaPhiM(tree.j1pt, tree.j1eta, tree.j1phi,
                                      tree.j1m)
                    j2p4 = ROOT.TLorentzVector(0, 0, 0, 0)
                    j2p4.SetPtEtaPhiM(tree.j2pt, tree.j2eta, tree.j2phi,
                                      tree.j2m)
                    zj2b = (j1p4 + j2p4).Pt() - boson.Pt()

        #proton tracks (standard and mixed)
        ev_pos_protons, ev_neg_protons = [[], [], []], [[], [], []]
        ppsPosEff, ppsPosEffUnc = 1.0, 0.0
        ppsNegEff, ppsNegEffUnc = 1.0, 0.0
        if isSignal or (isData and isRPIn):
            ev_pos_protons, ev_neg_protons = getTracksPerRomanPot(
                tree, minCsi=MINCSI)
            orig_ev_pos_protons = copy.deepcopy(ev_pos_protons)
            orig_ev_neg_protons = copy.deepcopy(ev_neg_protons)

        #if data and there is nothing to mix store the main characteristics of the event and continue
        if evMixTool.isIdle():
            if isData and isRPIn:
                if (isZ and tree.evcat == DIMUONS
                        and boson.Pt() < 10) or evcat == 'em':
                    rpDataKey = (evEra, beamXangle, int(tree.evcat))
                    if not rpDataKey in rpData: rpData[rpDataKey] = []
                    rpData[rpDataKey].append(
                        MixedEventSummary(puDiscr=[
                            len(extra_muons), nvtx, rho, PFMultSumHF,
                            PFHtSumHF, PFPzSumHF, rfc
                        ],
                                          pos_protons=ev_pos_protons,
                                          neg_protons=ev_neg_protons))
            continue

        #event mixing
        mixed_pos_protons, mixed_neg_protons, mixed_pudiscr = evMixTool.getNew(
            evEra=evEra,
            beamXangle=beamXangle,
            isData=isData,
            validAngles=VALIDLHCXANGLES,
            mixEvCategs=[DIMUONS, EMU])
        ppsEff, ppsEffUnc = 1.0, 0.0
        if isSignal:

            ppsPosEff, ppsPosEffUnc = 0.0, 0.0
            if len(ev_pos_protons[2]) > 0:
                ppsPosEff, ppsPosEffUnc = ppsEffReader.getPPSEfficiency(
                    evEra, beamXangle, ev_pos_protons[2][0], rp=3)

            ppsNegEff, ppsNegEffUnc = 0.0, 0.0
            if len(ev_neg_protons[2]) > 0:
                ppsNegEff, ppsNegEffUnc = ppsEffReader.getPPSEfficiency(
                    evEra, beamXangle, ev_neg_protons[2][0], rp=103)

            rawSigHyp = 0
            if len(ev_neg_protons[1]) > 0: rawSigHyp += 1
            if len(ev_neg_protons[0]) > 0: rawSigHyp += 2
            if len(ev_pos_protons[1]) > 0: rawSigHyp += 4
            if len(ev_pos_protons[0]) > 0: rawSigHyp += 8

            #assign the final list of reconstructed protons depending on how the sighyp is requested
            ev_pos_protons, ev_neg_protons, ppsEff, ppsEffUnc = ppsEffReader.getProjectedFinalState(
                ev_pos_protons, ppsPosEff, ppsPosEffUnc, ev_neg_protons,
                ppsNegEff, ppsNegEffUnc, sighyp)
            #mixed_pos_protons={DIMUONS:ev_pos_protons,EMU:ev_pos_protons}
            #mixed_neg_protons={DIMUONS:ev_neg_protons,EMU:ev_neg_protons}
            mixed_pos_protons, mixed_neg_protons = evMixTool.mergeWithMixedEvent(
                ev_pos_protons, mixed_pos_protons, ev_neg_protons,
                mixed_neg_protons)

            orig_mixed_pos_protons, orig_mixed_neg_protons = evMixTool.mergeWithMixedEvent(
                orig_ev_pos_protons, mixed_pos_protons, orig_ev_neg_protons,
                mixed_neg_protons)

            #control before and after projection
            ht.fill((rawSigHyp, sighyp, 1.0), 'sighyp', ['raw'])
            ht.fill((rawSigHyp, sighyp, ppsEff), 'sighyp', ['wgt'])

            n_extra_mu, nvtx, rho, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc = mixed_pudiscr[
                DIMUONS]

        #kinematics using RP tracks
        pos_protons = ev_pos_protons if isData else mixed_pos_protons[DIMUONS]
        neg_protons = ev_neg_protons if isData else mixed_neg_protons[DIMUONS]
        proton_cat, csi_pos, csi_neg, ppSystem, mmassSystem = getDiProtonCategory(
            pos_protons, neg_protons, boson, ALLOWPIXMULT)

        #compare categorization with fully exclusive selection of pixels
        ht.fill((0, ppsEff), 'catcount', ['inc'])
        if len(pos_protons[1]) in ALLOWPIXMULT and len(
                neg_protons[1]) in ALLOWPIXMULT:
            ht.fill((1, ppsEff), 'catcount', ['inc'])
        ht.fill((proton_cat + 1, ppsEff), 'catcount', ['inc'])
        if isSignal:
            ht.fill((0, 1.), 'catcount', ['single'])
            if len(orig_mixed_pos_protons[DIMUONS][1]) in ALLOWPIXMULT and len(
                    orig_mixed_neg_protons[DIMUONS][1]) in ALLOWPIXMULT:
                ht.fill((1, 1.), 'catcount', ['single'])

        #event categories
        cats = []
        cats.append(evcat)
        if isRPIn:
            cats += [evcat + 'rpin']
            if proton_cat > 0:
                cats += [evcat + 'rpinhpur']

        #fill control plots (for signal correct wgt by ee efficiency curve and duplicate for mm channel)
        wgt = tree.evwgt
        finalPlots = [[wgt, cats]]
        gen_pzpp = 0
        gen_pzwgt = [1., 1., 1.]
        gen_csiPos = 0.
        gen_csiNeg = 0.
        if isSignal:
            true_pos_protons, true_neg_protons = getTracksPerRomanPot(
                tree, True)
            if len(true_pos_protons[0]) > 0:
                gen_csiPos = true_pos_protons[0][0]
            if len(true_neg_protons[0]) > 0:
                gen_csiNeg = true_neg_protons[0][0]

            gen_pzpp = tree.gen_pzpp
            pzwid = 0.391 * gen_mX + 624
            gen_pzwgt[0] = ROOT.TMath.Gaus(gen_pzpp, 0, pzwid)
            gen_pzwgt[1] = ROOT.TMath.Gaus(gen_pzpp, 0,
                                           pzwid * 1.1) / gen_pzwgt[0]
            gen_pzwgt[2] = ROOT.TMath.Gaus(gen_pzpp, 0,
                                           pzwid * 0.9) / gen_pzwgt[0]

            #use the sum of pz weighted events as normalization factor
            if not isFullSimSignal:
                if isZ:
                    finalPlots = [[
                        wgt * ppsEff * gen_pzwgt[0] *
                        mcEff['eez'].Eval(boson.Pt()) / nSignalWgtSum, cats
                    ],
                                  [
                                      wgt * ppsEff * gen_pzwgt[0] *
                                      mcEff['mmz'].Eval(boson.Pt()) /
                                      nSignalWgtSum,
                                      [
                                          c.replace(evcat, 'mm') for c in cats
                                          if c[0:2] == 'ee'
                                      ]
                                  ]]

                    #reject Z->ee if one electron in the transition
                    if hasEEEBTransition:
                        finalPlots[0][0] = 0.

                elif isPhotonSignal:
                    finalPlots = [[
                        wgt * ppsEff * gen_pzwgt[0] *
                        mcEff['a'].Eval(boson.Pt()) / nSignalWgtSum, cats
                    ]]
            else:
                finalPlots = [[
                    wgt * ppsEff * gen_pzwgt[0] / nSignalWgtSum, cats
                ]]

        for pwgt, pcats in finalPlots:

            #fill plots only with fiducial signal contribution
            if isSignal and not isSignalFiducial(gen_csiPos, gen_csiNeg,
                                                 tree.gen_pzpp):
                continue

            #boson kinematics
            ht.fill((l1p4.Pt(), pwgt), 'l1pt', pcats)
            ht.fill((l2p4.Pt(), pwgt), 'l2pt', pcats)
            ht.fill((abs(l1p4.Eta()), pwgt), 'l1eta', pcats)
            ht.fill((abs(l2p4.Eta()), pwgt), 'l2eta', pcats)
            ht.fill((acopl, pwgt), 'acopl', pcats)
            ht.fill((boson.M(), pwgt), 'mll', pcats)
            ht.fill((boson.M(), pwgt), 'mll_full', pcats)
            ht.fill((boson.Rapidity(), pwgt), 'yll', pcats)
            ht.fill((boson.Eta(), pwgt), 'etall', pcats)
            ht.fill((boson.Pt(), pwgt), 'ptll', pcats)
            ht.fill((boson.Pt(), pwgt), 'ptll_high', pcats)
            ht.fill((costhetacs, pwgt), 'costhetacs', pcats)

            #pileup related
            ht.fill((beamXangle, pwgt), 'xangle', pcats)
            ht.fill((nvtx, pwgt), 'nvtx', pcats)
            ht.fill((rho, pwgt), 'rho', pcats)
            ht.fill((met, pwgt), 'met', pcats)
            ht.fill((mpf, pwgt), 'mpf', pcats)
            ht.fill((njets, pwgt), 'njets', pcats)
            if njets > 0: ht.fill((zjb, pwgt), 'zjb', pcats)
            if njets > 1: ht.fill((zj2b, pwgt), 'zj2b', pcats)
            ht.fill((nch, pwgt), 'nch', pcats)
            #ht.fill((getattr(tree,'rfc_%d'%beamXangle),pwgt), 'rfc',         pcats)
            ht.fill((PFMultSumHF, pwgt), 'PFMultHF', pcats)
            ht.fill((PFHtSumHF, pwgt), 'PFHtHF', pcats)
            ht.fill((PFPzSumHF / 1.e3, pwgt), 'PFPZHF', pcats)
            ht.fill((n_extra_mu, pwgt), 'nextramu', pcats)
            if isFullSimSignal or not isSignal:
                ht.fill((tree.metfilters, pwgt), 'metbits', pcats)
                for sd in ['HE', 'EE', 'EB']:
                    ht.fill((getattr(tree, 'PFMultSum' + sd), pwgt),
                            'PFMult' + sd, pcats)
                    ht.fill((getattr(tree, 'PFHtSum' + sd), pwgt), 'PFHt' + sd,
                            pcats)
                    ht.fill((getattr(tree, 'PFPzSum' + sd) / 1.e3, pwgt),
                            'PFPZ' + sd, pcats)
                for mp4 in extra_muons:
                    ht.fill((mp4.Pt(), pwgt), 'extramupt', pcats)
                    ht.fill((abs(mp4.Eta()), pwgt), 'extramueta', pcats)

            #proton counting and kinematics
            for ip in range(3):
                for irp, rpside in [(0, '%dpos' % ip), (1, '%dneg' % ip)]:
                    csiColl = pos_protons[ip] if irp == 0 else neg_protons[ip]
                    ht.fill((len(csiColl), pwgt), 'ntk', pcats, rpside)
                    for csi in csiColl:
                        ht.fill((csi, pwgt), 'csi', pcats, rpside)

            #diproton kinematics
            if proton_cat < 0:
                ht.fill((0, pwgt), 'ppcount', pcats)
            else:
                ht.fill((1, pwgt), 'ppcount', pcats)
                ht.fill((ppSystem.M(), pwgt), 'mpp', pcats)
                ht.fill((ppSystem.Pz(), pwgt), 'pzpp', pcats)
                ht.fill((ppSystem.Rapidity(), pwgt), 'ypp', pcats)
                mmass = mmassSystem.M()
                ht.fill((mmass, pwgt), 'mmass_full', pcats)
                ht.fill((mmass, pwgt), 'mmass_full', pcats, '%d' % proton_cat)
                if mmass > 0:
                    ht.fill((mmass, pwgt), 'mmass', pcats)
                    ht.fill((mmass, pwgt), 'mmass', pcats, '%d' % proton_cat)

            #signal characteristics in the absense of pileup
            if isSignal:
                nopu_proton_cat, nopu_csi_pos, nopu_csi_neg, nopu_ppSystem, nopu_mmassSystem = getDiProtonCategory(
                    ev_pos_protons, ev_neg_protons, boson, ALLOWPIXMULT)
                if nopu_proton_cat > 0:
                    nopu_mmass = nopu_mmassSystem.M()
                    ht.fill((nopu_ppSystem.M(), pwgt), 'mpp', pcats, 'nopu')
                    ht.fill((nopu_mmass, pwgt), 'mmass_full', pcats, 'nopu')
                    ht.fill((nopu_mmass, pwgt), 'mmass_full', pcats,
                            '%dnopu' % nopu_proton_cat)
                    if nopu_mmass > 0:
                        ht.fill((nopu_mmass, pwgt), 'mmass', pcats, 'nopu')
                        ht.fill((nopu_mmass, pwgt), 'mmass', pcats,
                                '%dnopu' % nopu_proton_cat)

        if not isData and not isSignal and not isDY: continue

        #save the event summary for the statistical analysis
        nMixTries = 100 if isData else 1
        for itry in range(2 * nMixTries + 1):

            itry_wgt = wgt

            #nominal
            if itry == 0:
                mixType = 0
                i_pos_protons = copy.deepcopy(pos_protons)
                i_neg_protons = copy.deepcopy(neg_protons)

                #shift csi by 1%
                i_pos_protons_syst = []
                i_neg_protons_syst = []
                for ialgo in range(3):
                    i_pos_protons_syst.append(
                        [1.01 * x for x in pos_protons[ialgo]])
                    i_neg_protons_syst.append(
                        [1.01 * x for x in neg_protons[ialgo]])

            else:
                #get a new event to mix
                i_mixed_pos_protons, i_mixed_neg_protons, i_mixed_pudiscr = evMixTool.getNew(
                    evEra=evEra,
                    beamXangle=beamXangle,
                    isData=isData,
                    validAngles=VALIDLHCXANGLES,
                    mixEvCategs=[DIMUONS, EMU])

                #FIXME this is broken in this new version
                #if MIXEDRPSIG:
                #    sigCsi=random.choice( MIXEDRPSIG[beamXangle] )
                #    for mixEvCat in mixed_far_rptks:
                #        tksPos=mixed_far_rptks[mixEvCat][0]+[sigCsi[0]]
                #        shuffle(tksPos)
                #        tksNeg=mixed_far_rptks[mixEvCat][1]+[sigCsi[1]]
                #        shuffle(tksNeg)
                #        mixed_far_rptks[mixEvCat]=(tksPos,tksNeg)

                #merge signal protons with pileup protons for first attempt
                if isSignal and itry == 1:
                    i_mixed_pos_protons, i_mixed_neg_protons = evMixTool.mergeWithMixedEvent(
                        ev_pos_protons, i_mixed_pos_protons, ev_neg_protons,
                        i_mixed_neg_protons)
                    if not isFullSimSignal:
                        n_extra_mu, nvtx, rho, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc = i_mixed_pudiscr[
                            DIMUONS]

                itry_wgt = wgt / float(nMixTries)

                if itry <= nMixTries or isSignal:
                    mixType = 1
                    if isSignal: mixType = itry
                    i_pos_protons = i_mixed_pos_protons[DIMUONS]
                    i_neg_protons = i_mixed_neg_protons[DIMUONS]
                    i_pos_protons_syst = i_mixed_pos_protons[EMU]
                    i_neg_protons_syst = i_mixed_neg_protons[EMU]
                else:
                    mixType = 2
                    i_pos_protons = i_mixed_pos_protons[DIMUONS]
                    i_neg_protons = copy.deepcopy(neg_protons)
                    i_pos_protons_syst = copy.deepcopy(pos_protons)
                    i_neg_protons_syst = i_mixed_neg_protons[DIMUONS]

            i_proton_cat, i_csi_pos, i_csi_neg, i_ppSystem, i_mmassSystem = getDiProtonCategory(
                i_pos_protons, i_neg_protons, boson, ALLOWPIXMULT)
            i_proton_cat_syst, i_csi_pos_syst, i_csi_neg_syst, i_ppSystem_syst, i_mmassSystem_syst = getDiProtonCategory(
                i_pos_protons_syst, i_neg_protons_syst, boson, ALLOWPIXMULT)

            #if itry>nMixTries:
            #    print itry,mixType
            #    print '\t',i_pos_protons,i_pos_protons_syst
            #    print '\t--->',i_proton_cat,     i_csi_pos,     i_csi_neg
            #    print '\t',i_neg_protons,i_neg_protons_syst
            #    print '\t--->',i_proton_cat_syst,i_csi_pos_syst,i_csi_neg_syst

            passAtLeastOneSelection = (i_proton_cat > 0
                                       or i_proton_cat_syst > 0)

            #start event summary
            evSummary.reset()
            evSummary.sighyp[0] = int(sighyp)
            if isData:
                evSummary.run[0] = int(tree.run)
                evSummary.event[0] = long(tree.event)
                evSummary.lumi[0] = int(tree.lumi)

            evSummary.era[0] = int(ord(evEra[-1]))
            evSummary.cat[0] = int(tree.evcat)
            evSummary.isOffZ[0] = int(isOffZ)
            evSummary.wgt[0] = itry_wgt
            evSummary.xangle[0] = int(beamXangle)
            evSummary.l1pt[0] = l1p4.Pt()
            evSummary.l1eta[0] = l1p4.Eta()
            evSummary.l2pt[0] = l2p4.Pt()
            evSummary.l2eta[0] = l2p4.Eta()
            evSummary.bosonm[0] = boson.M()
            evSummary.bosonpt[0] = boson.Pt()
            evSummary.bosoneta[0] = boson.Eta()
            evSummary.bosony[0] = boson.Rapidity()
            evSummary.acopl[0] = acopl
            evSummary.costhetacs[0] = costhetacs
            evSummary.njets[0] = int(njets)
            evSummary.mpf[0] = mpf
            evSummary.zjb[0] = zjb
            evSummary.zj2b[0] = zj2b
            evSummary.nch[0] = int(nch)
            evSummary.nvtx[0] = int(nvtx)
            evSummary.rho[0] = rho
            evSummary.PFMultSumHF[0] = int(PFMultSumHF)
            evSummary.PFHtSumHF[0] = PFHtSumHF
            evSummary.PFPzSumHF[0] = PFPzSumHF
            evSummary.rfc[0] = rfc
            evSummary.gen_pzpp[0] = gen_pzpp
            evSummary.gen_pzwgtUp[0] = gen_pzwgt[1]
            evSummary.gen_pzwgtDown[0] = gen_pzwgt[2]
            evSummary.gencsi1[0] = gen_csiPos
            evSummary.gencsi2[0] = gen_csiNeg

            #vary boson energy scale
            if i_ppSystem:
                boson_up = boson * 1.03
                evSummary.mmissvup[0] = buildMissingMassSystem(
                    i_ppSystem, boson_up).M()
                boson_dn = boson * 0.97
                evSummary.mmissvdn[0] = buildMissingMassSystem(
                    i_ppSystem, boson_dn).M()

            evSummary.mixType[0] = mixType
            evSummary.protonCat[0] = i_proton_cat
            if i_proton_cat > 0:
                evSummary.csi1[0] = i_csi_pos
                evSummary.csi2[0] = i_csi_neg
                evSummary.mpp[0] = i_ppSystem.M()
                evSummary.ypp[0] = i_ppSystem.Rapidity()
                evSummary.pzpp[0] = i_ppSystem.Pz()
                evSummary.mmiss[0] = i_mmassSystem.M()
                evSummary.ymmiss[0] = i_mmassSystem.Rapidity()
                evSummary.ppsEff[0] = ppsEff
                evSummary.ppsEffUnc[0] = ppsEffUnc

            evSummary.systprotonCat[0] = i_proton_cat_syst
            if i_proton_cat_syst > 0:
                evSummary.systcsi1[0] = i_csi_pos_syst
                evSummary.systcsi2[0] = i_csi_neg_syst
                evSummary.systmpp[0] = i_ppSystem_syst.M()
                evSummary.systypp[0] = i_ppSystem_syst.Rapidity()
                evSummary.systpzpp[0] = i_ppSystem_syst.Pz()
                evSummary.systmmiss[0] = i_mmassSystem_syst.M()
                evSummary.systymmiss[0] = i_mmassSystem_syst.Rapidity()
                evSummary.systppsEff[0] = ppsEff
                evSummary.systppsEffUnc[0] = ppsEffUnc

            #if no selection passes the cuts ignore its summary
            if not passAtLeastOneSelection: continue

            #for signal update the event weight for ee/mm/photon hypothesis
            if isData or isDY:
                tOut.Fill()

            elif isFullSimSignal:
                origWgt = evSummary.wgt[0]
                evSummary.wgt[0] = origWgt * gen_pzwgt[0] / nSignalWgtSum
                tOut.Fill()

            elif isSignal:

                origWgt = evSummary.wgt[0]

                if isZ:
                    #add a copy for ee
                    if not hasEEEBTransition:
                        evSummary.cat[0] = DIELECTRONS
                        evSummary.wgt[0] = origWgt * gen_pzwgt[0] * mcEff[
                            'eez'].Eval(boson.Pt()) / nSignalWgtSum
                        tOut.Fill()

                    #add a copy for mm
                    evSummary.cat[0] = DIMUONS
                    evSummary.wgt[0] = origWgt * gen_pzwgt[0] * mcEff[
                        'mmz'].Eval(boson.Pt()) / nSignalWgtSum
                    tOut.Fill()

                if isA:
                    #add a copy for the photon
                    evSummary.cat[0] = SINGLEPHOTON
                    evSummary.wgt[0] = origWgt * gen_pzwgt[0] * mcEff[
                        'a'].Eval(boson.Pt()) / nSignalWgtSum
                    tOut.Fill()

    #dump events for the mixing
    nSelRPData = sum([len(rpData[x]) for x in rpData])
    if nSelRPData > 0:
        rpDataOut = outFileName.replace('.root', '.pck')
        print 'Saving', nSelRPData, 'events for mixing in', rpDataOut
        with open(rpDataOut, 'w') as cachefile:
            pickle.dump(rpData, cachefile, pickle.HIGHEST_PROTOCOL)

    #if there was no mixing don't do anything else
    if not MIXEDRP: return

    #save results
    fOut.cd()
    tOut.Write()
    ht.writeToFile(fOut)
    fOut.Close()
Exemple #4
0
def buildControlPlots(tag, excSel):
    """loop over data events and make control plots"""

    baseDir = '/eos/cms/store/cmst3/user/psilva/ExclusiveAna/ab05162/Chunks/'

    if len(tag) == 1:
        tree = ROOT.TChain('tree')
        for f in os.listdir(baseDir):
            if not 'Data13TeV_2017%s_DoubleMuon' % tag in f: continue
            tree.AddFile(os.path.join(baseDir, f))
    elif 'MC13TeV_ppxz_m' in tag:
        tree = ROOT.TChain('analysis/data')
        for x in [120, 130, 140, 150]:
            tree.AddFile(os.path.join(baseDir, '%s_x%d_0.root' % (tag, x)))

    nentries = tree.GetEntries()
    print 'Analysing', nentries, 'events for tag', tag
    if nentries == 0:
        return

    #book histograms
    ht = HistoTool()
    ht.add(ROOT.TH1F('n', 'RP;Proton multiplicity;PDF', 5, 0, 5))
    ht.add(ROOT.TH1F('csi', ';#xi;PDF', 50, 0, 0.3))
    ht.add(ROOT.TH2F('csi2d', ';#xi(1);#xi(2);PDF', 50, 0, 0.3, 50, 0, 0.3))
    ht.add(ROOT.TH1F('mpp', ';m_{pp} [GeV];PDF', 100, 0, 2500))
    ht.add(
        ROOT.TH2F('mpp2d', ';m_{pp}(far) [GeV];m_{pp}(near) PDF', 100, 0, 2500,
                  100, 0, 2500))
    ht.add(
        ROOT.TH1F('dmpp', 'Near-Far;m_{pp}(near)-m_{pp}(far) [GeV];PDF', 100,
                  -200, 200))
    ht.add(ROOT.TH1F('mmass', ';Missing mass [GeV];Events', 100, -1000, 3000))
    ht.add(
        ROOT.TH2F('mmass2d',
                  ';Missing mass (near) [GeV];Missing mass (far);PDF', 100,
                  -1000, 3000, 100, -1000, 3000))
    ht.add(
        ROOT.TH1F('dmmass', 'Near-Far;#Delta missing mass [GeV];Events', 100,
                  -200, 200))
    ht.add(ROOT.TH1F('nvtx', ';Vertex multiplicity;Events', 50, 0, 50))
    ht.add(
        ROOT.TH1F('xangle', ';LHC crossing angle [#murad];Events', 4, 120,
                  160))
    ht.add(ROOT.TH1F('phiboson', ';#phi(boson);Events', 50, -3.15, 3.15))
    ht.add(ROOT.TH1F('yboson', ';Boson rapidity;Events', 50, -2, 2))
    ht.add(ROOT.TH1F('dyboson', ';y(boson)-y(central);Events', 50, -2, 2))

    for i in range(nentries):
        tree.GetEntry(i)
        if not tree.isZ: continue
        if tree.bosonpt > 10: continue
        if i % 1000 == 0: drawProgressBar(float(i) / float(nentries))

        #get basic information from the event
        boson = ROOT.TLorentzVector(0, 0, 0, 0)
        boson.SetPtEtaPhiM(tree.bosonpt, tree.bosoneta, tree.bosonphi,
                           tree.mboson)
        rp023, rp123 = getTracksPerRomanPot(tree)
        rp003, rp103 = getTracksPerRomanPot(tree, False, False)

        #build up category flags
        passPix = False
        passStrip = False
        passMatch = False
        csi_rp023, csi_rp123 = -1, -1
        csi_rp003, csi_rp103 = -1, 1
        if excSel:
            if len(rp023) == 1 and len(rp123) == 1:
                passPix = True
                csi_rp023 = rp023[0]
                csi_rp123 = rp123[0]
            if len(rp003) == 1 and len(rp103) == 1:
                passStrip = True
                csi_rp003 = rp003[0]
                csi_rp103 = rp103[0]

        else:

            passPix = True if len(rp023) > 0 and len(rp123) > 0 else False
            passStrip = True if len(rp003) == 1 and len(rp103) == 1 else False

            #take the max. csi for pixels as starting point
            csi_rp023 = max(rp023) if len(rp023) > 0 else -1
            csi_rp123 = max(rp123) if len(rp123) > 0 else -1

            #if matching to strips is found use that instead
            if len(rp003) == 1:
                csi_rp003 = rp003[0]
                for x in rp023:
                    if abs(csi_rp003 - x) > 0.02: continue
                    csi_rp023 = x
                    break
            if len(rp103) == 1:
                csi_rp103 = rp103[0]
                for x in rp123:
                    if abs(csi_rp103 - x) > 0.02: continue
                    csi_rp123 = x
                    break

        if passPix and passStrip:
            if abs(csi_rp003 - csi_rp023) < 0.02:
                if abs(csi_rp103 - csi_rp123) < 0.02:
                    passMatch = True

        #diproton kinematics
        pp_far = buildDiproton([[csi_rp023], [csi_rp123]]) if passPix else None
        mmass_far = (pp_far - boson).M() if pp_far else None
        pp_near = buildDiproton([[csi_rp003], [csi_rp103]
                                 ]) if passStrip else None
        mmass_near = (pp_near - boson).M() if pp_near else None

        #categories
        cats = ['inc']
        if passPix: cats += ['passPix']
        if passStrip: cats += ['passStrip']
        if passPix and passStrip: cats += ['passPixandpassStrip']
        if passPix and passStrip and passMatch:
            cats += ['passPixandpassStripmatched']
        if passStrip and passMatch: cats += ['passStripmatched']
        if passPix and passMatch: cats += ['passPixmatched']

        #global variables
        ht.fill((tree.nvtx, 1), 'nvtx', cats)
        ht.fill((tree.beamXangle, 1), 'xangle', cats)
        ht.fill((boson.Rapidity(), 1), 'yboson', cats)
        ht.fill((boson.Phi(), 1), 'phiboson', cats)
        ht.fill((boson.Pt(), 1), 'ptboson', cats)

        #individual RP plots
        for rpinfo, rp in [(rp003, 'RP003'), (rp023, 'RP023'),
                           (rp103, 'RP103'), (rp123, 'RP123')]:
            ht.fill((len(rpinfo), 1), 'n', cats, rp)
            for x in rpinfo:
                ht.fill((x, 1), 'csi', cats, rp)

        #RP correlations
        for csi1, csi2, rpcor in [(csi_rp003, csi_rp023, 'pos'),
                                  (csi_rp103, csi_rp123, 'neg')]:
            if csi1 < 0 or csi2 < 0: continue
            ht.fill((csi1, csi2, 1), 'csi2d', cats, rpcor)

        #diproton kinematics
        for pp, mmass, csi0, csi1, pptag in [
            (pp_near, mmass_near, csi_rp003, csi_rp103, 'near'),
            (pp_far, mmass_far, csi_rp023, csi_rp123, 'far')
        ]:
            if pp is None: continue
            ht.fill((pp.M(), 1), 'mpp', cats, pptag)
            ht.fill((mmass, 1), 'mmass', cats, pptag)

            ycen = 0.5 * ROOT.TMath.Log(csi0 / csi1)
            ht.fill((boson.Rapidity() - ycen, 1), 'dyboson', cats)

        #diproton correlations
        if pp_near and pp_far:
            ht.fill((pp_near.M() - pp_far.M(), 1), 'dmpp', cats)
            ht.fill((pp_near.M(), pp_far.M(), 1), 'mpp2d', cats)
            ht.fill((mmass_near - mmass_far, 1), 'dmmass', cats)
            ht.fill((mmass_near, mmass_far, 1), 'mmass2d', cats)

    recTag = 'exc' if excSel else 'inc'
    outURL = 'RPcontrol_%s_era%s.root' % (recTag, tag)
    ht.writeToFile(outURL)
    print 'Results can be found in', outURL
def buildControlPlots(args):
    """loop over data events and fill some basic control plots based on Z->mumu pT<10 GeV data"""

    tag, ch = args
    baseDir = '/eos/cms/store/cmst3/user/psilva/ExclusiveAna/final/2017_unblind_multi/Chunks/'
    tree = ROOT.TChain('tree')
    for f in os.listdir(baseDir):
        if ch == 13 * 13 and not 'Data13TeV_2017%s_DoubleMuon' % tag in f:
            continue
        if ch == 11 * 11 and not 'Data13TeV_2017%s_DoubleEG' % tag in f:
            continue
        tree.AddFile(os.path.join(baseDir, f))

    nentries = tree.GetEntries()
    if nentries == 0: return
    print 'Analysing', nentries, 'events for tag', tag, 'ch=', ch

    #book histograms
    ht = HistoTool()
    ht.add(ROOT.TH1F('n', ';Proton multiplicity;PDF', 5, 0, 5))
    ht.add(ROOT.TH1F('csi', ';#xi;PDF', 50, 0, 0.3))
    ht.add(ROOT.TH1F('mpp', ';m_{pp} [GeV];PDF', 100, 0, 2500))
    ht.add(ROOT.TH1F('mmass', ';Missing mass [GeV];Events', 100, -1000, 3000))
    ht.add(
        ROOT.TH1F('xangle', ';Beam crossing angle [#murad];Events', 4, 120,
                  160))
    ht.add(ROOT.TH1F('nvtx', ';Vertex multiplicity;Events', 50, 0, 50))
    ht.add(ROOT.TH1F('ptll', ';Transverse momentum [GeV];Events', 20, 0, 10))
    ht.add(
        ROOT.TH1F('met', ';Missing transverse energy [GeV];Events', 20, 0,
                  200))
    ht.add(
        ROOT.TH1F('dphimetz',
                  ';#Delta#phi[E_{T}^{miss},p_{T}(ll)] [rad];Events', 20, 0,
                  3.15))
    ht.add(
        ROOT.TH1F('nch', ';Charged particle multiplicity;Events', 20, 0, 100))
    ht.add(ROOT.TH1F('rue', ';p_{T}(vtx)/p_{T}(ll)-1;Events', 40, -1, 1))

    for i in range(nentries):

        tree.GetEntry(i)

        if i % 1000 == 0: drawProgressBar(float(i) / float(nentries))
        if not tree.isZ: continue
        if abs(tree.l1id * tree.l2id) != ch: continue
        if tree.bosonpt > 10: continue

        #decode variables of interest
        boson = ROOT.TLorentzVector(0, 0, 0, 0)
        boson.SetPtEtaPhiM(tree.bosonpt, tree.bosoneta, tree.bosonphi,
                           tree.mboson)
        tkPos, tkNeg = getTracksPerRomanPot(tree)
        rue = tree.sumPVChPt / boson.Pt() - 1
        dphimetz = abs(ROOT.TVector2.Phi_mpi_pi(tree.met_phi - boson.Phi()))
        xangle = tree.beamXangle

        cats = ['inc']
        for ialgo in range(3):
            algoCat = ['multi']
            if ialgo == 1: algoCat = ['px']
            if ialgo == 2: algoCat = ['strip']

            #individual RP plots
            for csiList, side in [(tkPos[ialgo], 'pos'),
                                  (tkNeg[ialgo], 'neg')]:
                ht.fill((len(csiList), 1), 'n', algoCat, side)
                if len(csiList) == 0: continue
                ht.fill((csiList[0], 1), 'csi', algoCat, side)
                for x in csiList:
                    ht.fill((x, 1), 'csi', algoCat, side + '_inc')

            #combined PPS variables
            passSel = True if len(tkPos[ialgo]) > 0 and len(
                tkNeg[ialgo]) > 0 else False
            if not passSel: continue
            cats += algoCat

            csi0, csi1 = tkPos[ialgo][0], tkNeg[ialgo][0]
            pp = buildDiProton(csi0, csi1)
            ht.fill((pp.M(), 1), 'mpp', algoCat)

            mmass = (pp - boson).M()
            ht.fill((mmass, 1), 'mmass', algoCat)

        #central, global variables
        ht.fill((tree.nvtx, 1), 'nvtx', cats + ['a%d' % xangle])
        ht.fill((xangle, 1), 'xangle', cats)
        ht.fill((boson.Pt(), 1), 'ptll', cats)
        ht.fill((tree.nchPV, 1), 'nch', cats)
        ht.fill((rue, 1), 'rue', cats)
        ht.fill((tree.met_pt, 1), 'met', cats)
        ht.fill((dphimetz, 1), 'dphimetz', cats)

    outURL = 'RPcontrol_era%s_ch%d.root' % (tag, ch)
    ht.writeToFile(outURL)
    print 'Results can be found in', outURL
Exemple #6
0
def runExclusiveAnalysis(inFile,
                         outFileName,
                         runLumiList,
                         effDir,
                         maxEvents=-1):
    """event loop"""

    global MIXEDRP
    global MIXEDRPSIG

    isData = True if 'Data' in inFile else False
    isDY = isDYFile(inFile)
    isSignal = isSignalFile(inFile)
    isPhotonSignal = isPhotonSignalFile(inFile)
    gen_mX = signalMassPoint(inFile) if isSignal else 0.

    #bind main tree with pileup discrimination tree, if failed return
    tree = ROOT.TChain('analysis/data' if isSignal else 'tree')
    tree.AddFile(inFile)
    #try:
    #    pudiscr_tree=ROOT.TChain('pudiscr')
    #    baseName=os.path.basename(inFile)
    #    baseDir=os.path.dirname(inFile)
    #    pudiscr_file=os.path.join(baseDir,'pudiscr',baseName)
    #    if not os.path.isfile(pudiscr_file):
    #        raise ValueError(pudiscr_file+' does not exist')
    #    pudiscr_tree.AddFile(pudiscr_file)
    #    tree.AddFriend(pudiscr_tree)
    #    print 'Added pu tree for',inFile
    #except:
    #    #print 'Failed to add pu discrimination tree as friend for',inFile
    #    return

    #identify data-taking era
    era = None
    if isData:
        era = os.path.basename(inFile).split('_')[1]

    #check if it is signal and load
    signalPt = []
    mcEff = {}
    if isSignal:
        signalPt = [
            float(x) for x in re.findall(r'\d+', os.path.basename(inFile))[2:]
        ]
        for ch in ['eez', 'mmz', 'a']:
            effIn = ROOT.TFile.Open('%s/effsummary_%s_ptboson.root' %
                                    (effDir, ch))
            pname = 'gen%srec_ptboson_ZH#rightarrowllbb_eff' % ch
            if ch == 'a': pname = 'genarec_ptboson_EWK #gammajj_eff'
            mcEff[ch] = effIn.Get(pname)
            effIn.Close()

    #start event mixing tool
    evMixTool = EventMixingTool(mixedRP=MIXEDRP, validAngles=VALIDLHCXANGLES)

    #start histograms
    ht = HistoTool()

    #main analysis histograms
    ht.add(ROOT.TH1F('nvtx', ';Vertex multiplicity;Events', 50, 0, 100))
    ht.add(ROOT.TH1F('rho', ';Fastjet #rho;Events', 50, 0, 50))
    ht.add(
        ROOT.TH1F('xangle', ';LHC crossing angle [#murad];Events', 4, 120,
                  160))
    ht.add(ROOT.TH1F('mll', ';Invariant mass [GeV];Events', 50, 76, 106))
    ht.add(ROOT.TH1F('mll_full', ';Invariant mass [GeV];Events', 50, 0, 250))
    ht.add(ROOT.TH1F('yll', ';Rapidity;Events', 50, -3, 3))
    ht.add(ROOT.TH1F('etall', ';Pseudo-rapidity;Events', 50, -6, 6))
    ht.add(ROOT.TH1F('ptll', ';Transverse momentum [GeV];Events', 50, 0, 250))
    ht.add(
        ROOT.TH1F('ptll_high', ';Transverse momentum [GeV];Events', 50, 50,
                  500))
    ht.add(ROOT.TH1F('l1eta', ';Pseudo-rapidiy;Events', 50, 0, 2.5))
    ht.add(ROOT.TH1F('l1pt', ';Transverse momentum [GeV];Events', 50, 0, 250))
    ht.add(ROOT.TH1F('l2eta', ';Pseudo-rapidiy;Events', 50, 0, 2.5))
    ht.add(ROOT.TH1F('l2pt', ';Transverse momentum [GeV];Events', 50, 0, 250))
    ht.add(ROOT.TH1F('acopl', ';A=1-|#Delta#phi|/#pi;Events', 50, 0, 1))
    ht.add(ROOT.TH1F('costhetacs', ';cos#theta_{CS};Events', 50, -1, 1))

    #pileup control
    #ht.add(ROOT.TH1F('rfc',';Random forest classifier probability;Events',50,0,1))
    for d in ['HF', 'HE', 'EE', 'EB']:
        ht.add(
            ROOT.TH1F('PFMult' + d, ';PF multiplicity (%s);Events' % d, 50, 0,
                      1000))
        ht.add(
            ROOT.TH1F('PFHt' + d, ';PF HT (%s) [GeV];Events' % d, 50, 0, 1000))
        ht.add(
            ROOT.TH1F('PFPz' + d, ';PF P_{z} (%s) [TeV];Events' % d, 50, 0,
                      40))
    ht.add(
        ROOT.TH1F('met', ';Missing transverse energy [GeV];Events', 50, 0,
                  200))
    ht.add(ROOT.TH1F('mpf', ';MPF;Events', 50, -5, 5))
    ht.add(ROOT.TH1F('metbits', ';MET filters;Events', 124, 0, 124))
    ht.add(ROOT.TH1F('njets', ';Jet multiplicity;Events', 5, 0, 5))
    ht.add(ROOT.TH1F('zjb', ';Z-jet balance [GeV];Events', 50, -150, 150))
    ht.add(ROOT.TH1F('zj2b', ';Z-2 jets balance [GeV];Events', 50, -150, 150))
    ht.add(ROOT.TH1F('nch', ';Charged particle multiplicity;Events', 50, 0,
                     50))
    ht.add(ROOT.TH1F('nextramu', ';Additional muons ;Events', 10, 0, 10))
    ht.add(
        ROOT.TH1F('extramupt', ';Additional muon p_{T} [GeV] ;Events', 10, 0,
                  50))
    ht.add(
        ROOT.TH1F('extramueta', ';Additional muon pseudo-rapidty ;Events', 10,
                  0, 2.5))

    #RP control
    ht.add(
        ROOT.TH1F('mpp', ';Di-proton invariant mass [GeV];Events', 50, 0,
                  3000))
    ht.add(ROOT.TH1F('pzpp', ';Di-proton p_{z} [GeV];Events', 50, -750, 750))
    ht.add(ROOT.TH1F('ypp', ';Di-proton rapidity;Events', 50, -2.5, 2.5))
    ht.add(
        ROOT.TH2F(
            'mpp2d',
            ';Far di-proton invariant mass [GeV];Near di-proton invariant mass [GeV];Events',
            50, 0, 3000, 50, 0, 3000))
    ht.add(
        ROOT.TH2F('ypp2d',
                  ';Far di-proton rapidity;Near di-proton rapidity;Events', 50,
                  -3, 3, 50, -3, 3))
    ht.add(
        ROOT.TH1F('mmass_full', ';Missing mass [GeV];Events', 50, -1000, 3000))
    ht.add(ROOT.TH1F('mmass', ';Missing mass [GeV];Events', 50, 0, 3000))
    ht.add(ROOT.TH1F('ntk', ';Track multiplicity;Events', 5, 0, 5))
    ht.add(ROOT.TH1F('ppcount', ';pp candidates;Events', 3, 0, 3))
    ht.add(ROOT.TH1F('csi', ';#xi;Events', 50, 0, 0.3))
    ht.add(
        ROOT.TH2F('csi2d', ';#xi(far);#xi(near);Events', 50, 0, 0.3, 50, 0,
                  0.3))

    nEntries = tree.GetEntries()
    print '....analysing', nEntries, 'in', inFile, ', with output @', outFileName
    if maxEvents > 0:
        nEntries = min(maxEvents, nEntries)
        print '      will process', nEntries, 'events'

    #compute number of events weighted by target pz spectrum
    nSignalWgtSum = 0.
    if isSignal:
        print 'Checking how many events are in the fiducial RP area...'
        for i in xrange(0, nEntries):
            tree.GetEntry(i)
            nSignalWgtSum += ROOT.TMath.Gaus(tree.gen_pzpp, 0,
                                             0.391 * gen_mX + 624)
        print '...signal weight sum set to', nSignalWgtSum, ' from ', nEntries, 'raw events'

    #loop over events
    rpData = {}
    selEvents = []
    summaryVars = 'cat:wgt:xangle:'
    summaryVars += 'l1pt:l1eta:l2pt:l2eta:acopl:bosonpt:bosoneta:bosony:costhetacs:njets:mpf:zjb:zj2b:'
    summaryVars += 'nch:nvtx:rho:PFMultSumHF:PFHtSumHF:PFPzSumHF:rfc:gen_pzpp:gen_pzwgtUp:gen_pzwgtDown:'
    if isSignal: summaryVars += 'gencsi1:gencsi2:'
    for pfix in ['', 'syst']:
        summaryVars += '{0}csi1:{0}csi2:{0}nearcsi1:{0}nearcsi2:{0}mpp:{0}ypp:{0}pzpp:{0}mmiss:'.format(
            pfix)
    summaryVars += 'mmissvup:mmissvdn:mixType:'
    summaryVars = summaryVars.split(':')[0:-1]  #cut away last token
    nfail = [0, 0, 0]
    for i in xrange(0, nEntries):

        tree.GetEntry(i)

        if i % 1000 == 0:
            sys.stdout.write('\r [ %d/100 ] done' %
                             (int(float(100. * i) / float(nEntries))))

        if isSignal:
            if isPhotonSignal and tree.evcat != SINGLEPHOTON: continue
            if not isPhotonSignal and tree.evcat == SINGLEPHOTON: continue

        #base event selection
        if tree.evcat == DIELECTRONS and tree.isZ:
            evcat = 'ee'
        elif tree.evcat == EMU and not tree.isSS:
            evcat = 'em'
        elif tree.evcat == DIMUONS and tree.isZ:
            evcat = 'mm'
        elif tree.evcat == SINGLEPHOTON and (isPhotonSignal
                                             or tree.hasATrigger):
            evcat = "a"
        elif tree.evcat == 0 and tree.hasZBTrigger:
            evcat == 'zbias'
        else:
            nfail[0] += 1
            continue

        #assign data-taking era and crossing angle
        evEra = era
        beamXangle = tree.beamXangle
        if not isData:
            evEra = getRandomEra()
            if not isSignal:
                xbin = evMixTool.getRandomLHCCrossingAngle(
                    evEra=evEra, evCat=SINGLEPHOTON if tree.isA else DIMUONS)
                beamXangle = VALIDLHCXANGLES[xbin]

        #check if RP is in (MC assume true by default)
        isRPIn = False if isData else True
        if isData and beamXangle in VALIDLHCXANGLES and isValidRunLumi(
                tree.run, tree.lumi, runLumiList):
            isRPIn = True
        if not isRPIn: nfail[1] += 1

        #lepton kinematics
        l1p4 = ROOT.TLorentzVector(0, 0, 0, 0)
        l2p4 = ROOT.TLorentzVector(0, 0, 0, 0)
        costhetacs = 0
        acopl = 0
        if tree.evcat != SINGLEPHOTON and tree.evcat != 0:
            acopl = 1.0 - abs(ROOT.TVector2.Phi_mpi_pi(
                tree.l1phi - tree.l2phi)) / ROOT.TMath.Pi()
            l1p4.SetPtEtaPhiM(tree.l1pt, tree.l1eta, tree.l1phi, tree.ml1)
            l2p4.SetPtEtaPhiM(tree.l2pt, tree.l2eta, tree.l2phi, tree.ml2)
            costhetacs = computeCosThetaStar(l1p4, l2p4)

            #at this point order by pT
            if l1p4.Pt() < l2p4.Pt():
                l1p4, l2p4 = l2p4, l1p4

        #boson kinematics
        boson = ROOT.TLorentzVector(0, 0, 0, 0)
        boson.SetPtEtaPhiM(tree.bosonpt, tree.bosoneta, tree.bosonphi,
                           tree.mboson)
        isZ = tree.isZ
        isA = tree.isA

        #for the signal-electron hypothesis this cut needs to be applied
        hasEEEBTransition = False
        if isZ:
            if abs(l1p4.Eta()) > 1.4442 and abs(l1p4.Eta()) < 1.5660:
                hasEEEBTransition = True
            if abs(l2p4.Eta()) > 1.4442 and abs(l2p4.Eta()) < 1.5660:
                hasEEEBTransition = True

        #PU-related variables
        #for signal most of these will be overriden by mixing
        n_extra_mu, nvtx, nch, rho, met, njets, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
        mpf, zjb, zj2b = 0, 0, 0
        extra_muons = []
        if not isSignal:
            nvtx = tree.nvtx
            nch = tree.nchPV
            rho = tree.rho
            met = tree.met_pt
            metphi = tree.met_phi
            njets = tree.nj
            PFMultSumHF = tree.PFMultSumHF
            PFHtSumHF = tree.PFHtSumHF
            PFPzSumHF = tree.PFPzSumHF
            #rfc=getattr(tree,'rfc_%d'%beamXangle)
            for im in range(tree.nrawmu):
                mup4 = ROOT.TLorentzVector(0, 0, 0, 0)
                mup4.SetPtEtaPhiM(tree.rawmu_pt[im], tree.rawmu_eta[im] / 10.,
                                  tree.rawmu_phi[im] / 10., 0.105)
                if mup4.DeltaR(l1p4) < 0.05: continue
                if mup4.DeltaR(l2p4) < 0.05: continue
                extra_muons.append(ROOT.TLorentzVector(mup4))
            n_extra_mu = len(extra_muons)

            metp4 = ROOT.TLorentzVector(0, 0, 0, 0)
            metp4.SetPtEtaPhiM(met, 0, metphi, 0)
            mpf = 1. + (metp4.Px() * boson.Px() +
                        metp4.Py() * boson.Py()) / (boson.Pt()**2 + 1.0e-6)
            if njets > 0:
                zjb = tree.j1pt - boson.Pt()
                if njets > 1:
                    j1p4 = ROOT.TLorentzVector(0, 0, 0, 0)
                    j1p4.SetPtEtaPhiM(tree.j1pt, tree.j1eta, tree.j1phi,
                                      tree.j1m)
                    j2p4 = ROOT.TLorentzVector(0, 0, 0, 0)
                    j2p4.SetPtEtaPhiM(tree.j2pt, tree.j2eta, tree.j2phi,
                                      tree.j2m)
                    zj2b = (j1p4 + j2p4).Pt() - boson.Pt()

        #proton tracks (standard and mixed)
        far_rptks, near_rptks = [[], []], [[], []]
        if isSignal or (isData and isRPIn):
            far_rptks = getTracksPerRomanPot(tree, minCsi=MINCSI)
            near_rptks = getTracksPerRomanPot(tree, False, False)

        #if data and there is nothing to mix store the main characteristics of the event and continue
        if evMixTool.isIdle():
            if isData and isRPIn:

                #for Z->mm use 25% otherwise we have way too many events to do this efficiently
                if (isZ and tree.evcat == DIMUONS and boson.Pt() < 10
                        and random.random() < 0.25) or evcat == 'em':

                    rpDataKey = (evEra, beamXangle, int(tree.evcat))
                    if not rpDataKey in rpData: rpData[rpDataKey] = []
                    rpData[rpDataKey].append(
                        MixedEvent(beamXangle, [
                            len(extra_muons), nvtx, rho, PFMultSumHF,
                            PFHtSumHF, PFPzSumHF, rfc
                        ], far_rptks, near_rptks))
            continue

        #event mixing
        mixed_far_rptks, mixed_near_rptks, mixed_pudiscr = evMixTool.getNew(
            evEra=evEra,
            beamXangle=beamXangle,
            isData=isData,
            validAngles=VALIDLHCXANGLES,
            mixEvCategs=[DIMUONS, EMU])

        if isSignal:
            mixed_far_rptks, mixed_near_rptks = evMixTool.mergeWithMixedEvent(
                far_rptks, mixed_far_rptks, near_rptks, mixed_near_rptks)
            n_extra_mu, nvtx, rho, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc = mixed_pudiscr[
                DIMUONS]

        #kinematics using RP tracks
        far_protons = far_rptks if isData else mixed_far_rptks[DIMUONS]
        near_protons = near_rptks if isData else mixed_near_rptks[DIMUONS]
        highPur = passHighPurSelection(far_protons, boson)
        ppSystem = buildDiProton(far_protons)
        mmassSystem = buildMissingMassSystem(far_protons, boson)
        nearppSystem = buildDiProton(near_protons)
        nearmmassSystem = buildMissingMassSystem(near_protons, boson)

        #event categories
        cats = []
        cats.append(evcat)
        if isRPIn:
            cats += [evcat + 'rpin']
            if highPur:
                cats += [evcat + 'rpinhpur']
                if mmassSystem and mmassSystem.M() > 0:
                    cats += [evcat + 'rpinhpur%dxangle' % beamXangle]

        #fill control plots (for signal correct wgt by ee efficiency curve and duplicate for mm channel)
        wgt = tree.evwgt
        finalPlots = [[wgt, cats]]
        gen_pzpp = 0
        gen_pzwgt = [1., 1., 1.]
        gen_csiPos, gen_csiNeg = 0, 0
        if isSignal:
            true_rptks = getTracksPerRomanPot(tree, True)

            if len(true_rptks[0]) > 0: gen_csiPos = true_rptks[0][0]
            if len(true_rptks[1]) > 0: gen_csiNeg = true_rptks[1][0]

            gen_pzpp = tree.gen_pzpp
            pzwid = 0.391 * gen_mX + 624
            gen_pzwgt[0] = ROOT.TMath.Gaus(gen_pzpp, 0, pzwid)
            gen_pzwgt[1] = ROOT.TMath.Gaus(gen_pzpp, 0,
                                           pzwid * 1.1) / gen_pzwgt[0]
            gen_pzwgt[2] = ROOT.TMath.Gaus(gen_pzpp, 0,
                                           pzwid * 0.9) / gen_pzwgt[0]

            #use the sum of pz weighted events as normalization factor
            if isZ:
                finalPlots = [[
                    wgt * gen_pzwgt[0] * mcEff['eez'].Eval(boson.Pt()) /
                    nSignalWgtSum, cats
                ],
                              [
                                  wgt * gen_pzwgt[0] *
                                  mcEff['mmz'].Eval(boson.Pt()) /
                                  nSignalWgtSum,
                                  [
                                      c.replace(evcat, 'mm') for c in cats
                                      if c[0:2] == 'ee'
                                  ]
                              ]]

                #reject Z->ee if one electron in the transition
                if hasEEEBTransition:
                    finalPlots[0][0] = 0.

            elif isPhotonSignal:
                finalPlots = [[
                    wgt * gen_pzwgt[0] * mcEff['a'].Eval(boson.Pt()) /
                    nSignalWgtSum, cats
                ]]

        for pwgt, pcats in finalPlots:

            #fill plots only with fiducial signal contribution
            if isSignal and not isSignalFiducial(gen_csiPos, gen_csiNeg,
                                                 tree.gen_pzpp):
                continue

            #boson kinematics
            ht.fill((l1p4.Pt(), pwgt), 'l1pt', pcats)
            ht.fill((l2p4.Pt(), pwgt), 'l2pt', pcats)
            ht.fill((abs(l1p4.Eta()), pwgt), 'l1eta', pcats)
            ht.fill((abs(l2p4.Eta()), pwgt), 'l2eta', pcats)
            ht.fill((acopl, pwgt), 'acopl', pcats)
            ht.fill((boson.M(), pwgt), 'mll', pcats)
            ht.fill((boson.M(), pwgt), 'mll_full', pcats)
            ht.fill((boson.Rapidity(), pwgt), 'yll', pcats)
            ht.fill((boson.Eta(), pwgt), 'etall', pcats)
            ht.fill((boson.Pt(), pwgt), 'ptll', pcats)
            ht.fill((boson.Pt(), pwgt), 'ptll_high', pcats)
            ht.fill((costhetacs, pwgt), 'costhetacs', pcats)

            #pileup related
            ht.fill((beamXangle, pwgt), 'xangle', pcats)
            ht.fill((nvtx, pwgt), 'nvtx', pcats)
            ht.fill((rho, pwgt), 'rho', pcats)
            ht.fill((met, pwgt), 'met', pcats)
            ht.fill((mpf, pwgt), 'mpf', pcats)
            ht.fill((njets, pwgt), 'njets', pcats)
            if njets > 0: ht.fill((zjb, pwgt), 'zjb', pcats)
            if njets > 1: ht.fill((zj2b, pwgt), 'zj2b', pcats)
            ht.fill((nch, pwgt), 'nch', pcats)
            #ht.fill((getattr(tree,'rfc_%d'%beamXangle),pwgt), 'rfc',         pcats)
            ht.fill((PFMultSumHF, pwgt), 'PFMultHF', pcats)
            ht.fill((PFHtSumHF, pwgt), 'PFHtHF', pcats)
            ht.fill((PFPzSumHF / 1.e3, pwgt), 'PFPZHF', pcats)
            ht.fill((n_extra_mu, pwgt), 'nextramu', pcats)
            if not isSignal:
                ht.fill((tree.metfilters, pwgt), 'metbits', pcats)
                for sd in ['HE', 'EE', 'EB']:
                    ht.fill((getattr(tree, 'PFMultSum' + sd), pwgt),
                            'PFMult' + sd, pcats)
                    ht.fill((getattr(tree, 'PFHtSum' + sd), pwgt), 'PFHt' + sd,
                            pcats)
                    ht.fill((getattr(tree, 'PFPzSum' + sd) / 1.e3, pwgt),
                            'PFPZ' + sd, pcats)
                for mp4 in extra_muons:
                    ht.fill((mp4.Pt(), pwgt), 'extramupt', pcats)
                    ht.fill((abs(mp4.Eta()), pwgt), 'extramueta', pcats)

            #proton counting and kinematics
            if far_protons and len(far_protons) > 0:
                for irp, rpside in [(0, 'pos'), (1, 'neg')]:
                    ht.fill((len(far_protons[irp]), pwgt), 'ntk', pcats,
                            rpside)
                    for csi in far_protons[irp]:
                        ht.fill((csi, pwgt), 'csi', pcats, rpside)
                        if not near_protons: continue
                        if len(near_protons[irp]) == 0: continue
                        csi_near = near_protons[irp][0]
                        ht.fill((csi, csi_near, pwgt), 'csi2d', pcats, rpside)

            #diproton kinematics
            if not ppSystem:
                ht.fill((0, pwgt), 'ppcount', pcats)
            else:
                ht.fill((1, pwgt), 'ppcount', pcats)
                ht.fill((ppSystem.M(), pwgt), 'mpp', pcats)
                ht.fill((ppSystem.Pz(), pwgt), 'pzpp', pcats)
                ht.fill((ppSystem.Rapidity(), pwgt), 'ypp', pcats)
                if nearppSystem:
                    ht.fill((2, pwgt), 'ppcount', pcats)
                    ht.fill((ppSystem.M(), nearppSystem.M(), pwgt), 'mpp2d',
                            pcats)
                    ht.fill(
                        (ppSystem.Rapidity(), nearppSystem.Rapidity(), pwgt),
                        'ypp2d', pcats)

                mmass = mmassSystem.M()
                ht.fill((mmass, pwgt), 'mmass_full', pcats)
                if mmass > 0:
                    ht.fill((mmass, pwgt), 'mmass', pcats)

            #signal characteristics in the absense of pileup
            if isSignal:
                nopu_far_protons = far_rptks
                nopu_near_protons = near_rptks
                nopu_ppSystem = buildDiProton(nopu_far_protons)
                if nopu_ppSystem:
                    nopu_mmassSystem = buildMissingMassSystem(
                        nopu_far_protons, boson)
                    nopu_mmass = nopu_mmassSystem.M()
                    ht.fill((nopu_ppSystem.M(), pwgt), 'mpp', pcats, 'nopu')
                    ht.fill((nopu_mmass, pwgt), 'mmass_full', pcats, 'nopu')
                    if nopu_mmass > 0:
                        ht.fill((nopu_mmass, pwgt), 'mmass', pcats, 'nopu')

        if not isData and not isSignal and not isDY: continue

        #save the event summary for the statistical analysis
        nMixTries = 100 if isData else 1
        for itry in range(2 * nMixTries + 1):

            itry_wgt = wgt

            #nominal
            if itry == 0:
                mixType = 0
                far_protons = far_rptks
                near_protons = near_rptks

                #shift csi by 1%
                far_protons_syst = []
                near_protons_syst = []
                for iside in range(2):

                    if far_protons:
                        far_protons_syst.append(
                            [1.01 * x for x in far_protons[iside]])
                    else:
                        far_protons_syst.append([])

                    if near_protons:
                        near_protons_syst.append(
                            [1.01 * x for x in near_protons[iside]])
                    else:
                        near_protons_syst.append([])

            else:

                #get a new event to mix
                mixed_far_rptks, mixed_near_rptks, _ = evMixTool.getNew(
                    evEra=evEra,
                    beamXangle=beamXangle,
                    isData=isData,
                    validAngles=VALIDLHCXANGLES,
                    mixEvCategs=[DIMUONS, EMU])

                if MIXEDRPSIG:
                    sigCsi = random.choice(MIXEDRPSIG[beamXangle])
                    for mixEvCat in mixed_far_rptks:
                        tksPos = mixed_far_rptks[mixEvCat][0] + [sigCsi[0]]
                        shuffle(tksPos)
                        tksNeg = mixed_far_rptks[mixEvCat][1] + [sigCsi[1]]
                        shuffle(tksNeg)
                        mixed_far_rptks[mixEvCat] = (tksPos, tksNeg)

                #merge signal protons with pileup protons for first attempt
                if isSignal and itry == 1:
                    mixed_far_rptks, mixed_near_rptks = evMixTool.mergeWithMixedEvent(
                        far_rptks, mixed_far_rptks, near_rptks,
                        mixed_near_rptks)
                    n_extra_mu, nvtx, rho, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc = mixed_pudiscr[
                        DIMUONS]

                itry_wgt = wgt / float(nMixTries)

                if itry <= nMixTries or isSignal:
                    mixType = 1
                    if isSignal: mixType = itry
                    far_protons = mixed_far_rptks[DIMUONS]
                    far_protons_syst = mixed_far_rptks[EMU]
                    near_protons = mixed_near_rptks[DIMUONS]
                    near_protons_syst = mixed_near_rptks[EMU]
                else:
                    mixType = 2
                    far_protons = [mixed_far_rptks[DIMUONS][0], far_rptks[1]]
                    far_protons_syst = [
                        far_rptks[0], mixed_far_rptks[DIMUONS][1]
                    ]
                    near_protons = [
                        mixed_near_rptks[DIMUONS][0], near_rptks[1]
                    ]
                    near_protons_syst = [
                        near_rptks[0], mixed_near_rptks[DIMUONS][1]
                    ]

            evSummary = [
                tree.evcat, itry_wgt, beamXangle,
                l1p4.Pt(),
                l1p4.Eta(),
                l2p4.Pt(),
                l2p4.Eta(), acopl,
                boson.Pt(),
                boson.Eta(),
                boson.Rapidity(), costhetacs, njets, mpf, zjb, zj2b, nch, nvtx,
                rho, PFMultSumHF, PFHtSumHF, PFPzSumHF, rfc, gen_pzpp,
                gen_pzwgt[1], gen_pzwgt[2]
            ]

            if isSignal: evSummary += [gen_csiPos, gen_csiNeg]

            passAtLeastOneSelection = False

            #check if nominal passes the selection and and info to the event
            highPur = passHighPurSelection(far_protons, boson)
            ppSystem = buildDiProton(far_protons)
            mmassSystem = buildMissingMassSystem(far_protons, boson)

            #vary boson energy scale
            boson_up = boson * 1.03
            mmassSystem_vup = buildMissingMassSystem(far_protons, boson_up)
            boson_dn = boson * 0.97
            mmassSystem_vdn = buildMissingMassSystem(far_protons, boson_dn)

            if highPur and mmassSystem:
                passAtLeastOneSelection = True
                near_csiL = near_protons[0][0] if len(near_protons[0]) else 0
                near_csiR = near_protons[1][0] if len(near_protons[1]) else 0
                evSummary += [
                    far_protons[0][0], far_protons[1][0], near_csiL, near_csiR,
                    ppSystem.M(),
                    ppSystem.Rapidity(),
                    ppSystem.Pz(),
                    mmassSystem.M()
                ]
            else:
                evSummary += [0] * 8

            #repeat for systematics
            highPur_syst = passHighPurSelection(far_protons_syst, boson)
            ppSystem_syst = buildDiProton(far_protons_syst)
            mmassSystem_syst = buildMissingMassSystem(far_protons_syst, boson)

            if highPur_syst and mmassSystem_syst:
                passAtLeastOneSelection = True
                near_csiL = near_protons_syst[0][0] if len(
                    near_protons_syst[0]) else 0
                near_csiR = near_protons_syst[1][0] if len(
                    near_protons_syst[1]) else 0
                evSummary += [
                    far_protons_syst[0][0], far_protons_syst[1][0], near_csiL,
                    near_csiR,
                    ppSystem_syst.M(),
                    ppSystem_syst.Rapidity(),
                    ppSystem_syst.Pz(),
                    mmassSystem_syst.M()
                ]
            else:
                evSummary += [0] * 8

            #add information on the type of event (non-mix/mixed)
            evSummary += [
                mmassSystem_vup.M() if mmassSystem_vup else 0.,
                mmassSystem_vdn.M() if mmassSystem_vdn else 0., mixType
            ]

            #if no selection passes the cuts ignore its summary
            if not passAtLeastOneSelection: continue

            #for signal update the event weight for ee/mm/photon hypothesis
            if isData or isDY:
                selEvents.append(evSummary)
            elif isSignal:
                if isZ:
                    #add a copy for ee
                    if not hasEEEBTransition:
                        eeEvSummary = copy.copy(evSummary)
                        eeEvSummary[0] = DIELECTRONS
                        eeEvSummary[1] = evSummary[1] * gen_pzwgt[0] * mcEff[
                            'eez'].Eval(boson.Pt()) / nSignalWgtSum
                        selEvents.append(eeEvSummary)

                    #add a copy for mm
                    mmEvSummary = copy.copy(evSummary)
                    mmEvSummary[0] = DIMUONS
                    mmEvSummary[1] = evSummary[1] * gen_pzwgt[0] * mcEff[
                        'mmz'].Eval(boson.Pt()) / nSignalWgtSum
                    selEvents.append(mmEvSummary)

                if isA:
                    #add a copy for the photon
                    aEvSummary = copy.copy(evSummary)
                    aEvSummary[0] = SINGLEPHOTON
                    aEvSummary[1] = evSummary[1] * gen_pzwgt[0] * mcEff[
                        'a'].Eval(boson.Pt()) / nSignalWgtSum
                    selEvents.append(aEvSummary)

    #dump events for the mixing
    nSelRPData = sum([len(rpData[x]) for x in rpData])
    if nSelRPData > 0:
        rpDataOut = outFileName.replace('.root', '.pck')
        print 'Saving', nSelRPData, 'events for mixing in', rpDataOut
        with open(rpDataOut, 'w') as cachefile:
            pickle.dump(rpData, cachefile, pickle.HIGHEST_PROTOCOL)

    #if there was no mixing don't do anything else
    if not MIXEDRP: return

    #save results
    ht.writeToFile(outFileName)

    #dump events for fitting
    nSelEvents = len(selEvents)
    if nSelEvents > 0:
        print 'Adding', nSelEvents, 'selected events to', outFileName, '(', nfail, 'events failed baseline selection)'
        fOut = ROOT.TFile.Open(outFileName, 'UPDATE')
        fOut.cd()
        t = ROOT.TNtuple('data', 'data', ':'.join(summaryVars))
        for v in selEvents:
            t.Fill(array.array("f", v))
        t.Write()
        fOut.Close()