def __init__(self, tagger, year, workingpoint):
self._year = year
common = load('data/common.coffea')
self._wp = common['btagWPs'][tagger][year][workingpoint]
files = {
'deepflav': {
'2016': 'DeepJet_2016LegacySF_V1.csv',
'2017': 'DeepFlavour_94XSF_V4_B_F.csv',
'2018': 'DeepJet_102XSF_V1.csv'
},
'deepcsv': {
'2016': 'DeepCSV_2016LegacySF_V1.csv',
'2017': 'DeepCSV_94XSF_V5_B_F.csv',
'2018': 'DeepCSV_102XSF_V1.csv'
}
}
filename = 'data/' + files[tagger][year]
self.sf = BTagScaleFactor(filename, workingpoint)
files = {
'2016': 'btag2017.merged',
'2017': 'btag2017.merged',
'2018': 'btag2018.merged',
}
filename = 'hists/' + files[year]
btag = load(filename)
bpass = btag[tagger].integrate('dataset').integrate(
'wp', workingpoint).integrate('btag', 'pass').values()[()]
ball = btag[tagger].integrate('dataset').integrate(
'wp', workingpoint).integrate('btag').values()[()]
nom = bpass / np.maximum(ball, 1.)
self.eff = lookup_tools.dense_lookup.dense_lookup(
nom, [ax.edges() for ax in btag[tagger].axes()[3:]])
def __init__(self, year, workingpoint):
self._year = year
self._wp = BTagEfficiency.btagWPs[year][workingpoint]
files = {
'2016': 'DeepCSV_Moriond17_B_H.csv.gz',
'2017': 'DeepCSV_94XSF_V5_B_F.csv.gz',
'2018': 'DeepCSV_102XSF_V1.csv.gz',
}
filename = os.path.join(os.path.dirname(__file__), 'data', files[year])
self.sf = BTagScaleFactor(filename, workingpoint)
files = {
'2016': 'btagQCD2017.coffea',
'2017': 'btagQCD2017.coffea',
'2018': 'btagQCD2017.coffea',
}
filename = os.path.join(os.path.dirname(__file__), 'data', files[year])
btag = util.load(filename)
bpass = btag.integrate('btag', 'pass').values()[()]
ball = btag.integrate('btag').values()[()]
nom = bpass / numpy.maximum(ball, 1.)
dn, up = hist.clopper_pearson_interval(bpass, ball)
self.eff = dense_lookup(nom, [ax.edges() for ax in btag.axes()[1:]])
self.eff_statUp = dense_lookup(up,
[ax.edges() for ax in btag.axes()[1:]])
self.eff_statDn = dense_lookup(dn,
[ax.edges() for ax in btag.axes()[1:]])
def __init__(self, year):
self.year = year
if self.year == 2016:
pass
elif self.year == 2017:
pass
elif self.year == 2018:
SF_file = os.path.expandvars(
'$TWHOME/data/btag/DeepJet_102XSF_V2.csv')
self.btag_sf = BTagScaleFactor(SF_file, "medium", keep_df=False)
# and load the efficiencies
self.effs = {
'b':
Hist1D.from_json(
os.path.expandvars(
"$TWHOME/data/btag/Autumn18_b_eff_deepJet.json")),
'c':
Hist1D.from_json(
os.path.expandvars(
"$TWHOME/data/btag/Autumn18_c_eff_deepJet.json")),
'light':
Hist1D.from_json(
os.path.expandvars(
"$TWHOME/data/btag/Autumn18_light_eff_deepJet.json")),
}
class BTagCorrector:
def __init__(self, tagger, year, workingpoint):
self._year = year
common = load('data/common.coffea')
self._wp = common['btagWPs'][tagger][year][workingpoint]
files = {
'deepflav': {
'2016': 'DeepJet_2016LegacySF_V1.csv',
'2017': 'DeepFlavour_94XSF_V4_B_F.csv',
'2018': 'DeepJet_102XSF_V1.csv'
},
'deepcsv': {
'2016': 'DeepCSV_2016LegacySF_V1.csv',
'2017': 'DeepCSV_94XSF_V5_B_F.csv',
'2018': 'DeepCSV_102XSF_V1.csv'
}
}
filename = 'data/' + files[tagger][year]
self.sf = BTagScaleFactor(filename, workingpoint)
files = {
'2016': 'btag2017.merged',
'2017': 'btag2017.merged',
'2018': 'btag2018.merged',
}
filename = 'hists/' + files[year]
btag = load(filename)
bpass = btag[tagger].integrate('dataset').integrate(
'wp', workingpoint).integrate('btag', 'pass').values()[()]
ball = btag[tagger].integrate('dataset').integrate(
'wp', workingpoint).integrate('btag').values()[()]
nom = bpass / np.maximum(ball, 1.)
self.eff = lookup_tools.dense_lookup.dense_lookup(
nom, [ax.edges() for ax in btag[tagger].axes()[3:]])
def btag_weight(self, pt, eta, flavor, tag):
abseta = abs(eta)
#https://twiki.cern.ch/twiki/bin/viewauth/CMS/BTagSFMethods#1b_Event_reweighting_using_scale
def zerotag(eff):
return (1 - eff).prod()
eff = self.eff(flavor, pt, abseta)
sf_nom = self.sf.eval('central', flavor, abseta, pt)
sf_up = self.sf.eval('up', flavor, abseta, pt)
sf_down = self.sf.eval('down', flavor, abseta, pt)
eff_data_nom = np.minimum(1., sf_nom * eff)
eff_data_up = np.minimum(1., sf_up * eff)
eff_data_down = np.minimum(1., sf_down * eff)
nom = zerotag(eff_data_nom) / zerotag(eff)
up = zerotag(eff_data_up) / zerotag(eff)
down = zerotag(eff_data_down) / zerotag(eff)
if '-1' in tag:
nom = (1 - zerotag(eff_data_nom)) / (1 - zerotag(eff))
up = (1 - zerotag(eff_data_up)) / (1 - zerotag(eff))
down = (1 - zerotag(eff_data_down)) / (1 - zerotag(eff))
return np.nan_to_num(nom), np.nan_to_num(up), np.nan_to_num(down)
def validate_btag(filename, btagtype, etamax):
btagData = ROOT.BTagCalibration(btagtype, filename)
npts = 10000
flavor = numpy.full(npts, 5)
abseta = numpy.random.uniform(0, etamax, size=npts)
pt = numpy.random.exponential(50, size=npts) + numpy.random.exponential(20, size=npts)
pt = numpy.maximum(20.1, pt)
discr = numpy.random.rand(npts)
coffea_sf = BTagScaleFactor(filename, BTagScaleFactor.RESHAPE, 'iterativefit', keep_df=True)
btagReader = ROOT.BTagCalibrationReader(ROOT.BTagEntry.OP_RESHAPING, 'central', stdvec(['up_jes', 'down_jes']))
btagReader.load(btagData, ROOT.BTagEntry.FLAV_B, 'iterativefit')
btv_sf = makesf(btagReader)
for syst in ['central', 'up_jes', 'down_jes']:
csf = coffea_sf.eval(syst, flavor, abseta, pt, discr)
bsf = btv_sf(syst, flavor, abseta, pt, discr)
print(abs(csf - bsf).max())
flavor = numpy.random.choice([0, 4, 5], size=npts)
coffea_sf = BTagScaleFactor(filename, BTagScaleFactor.TIGHT, 'comb,mujets,incl', keep_df=True)
btagReader = ROOT.BTagCalibrationReader(ROOT.BTagEntry.OP_TIGHT, 'central', stdvec(['up', 'down']))
btagReader.load(btagData, ROOT.BTagEntry.FLAV_B, 'comb')
btagReader.load(btagData, ROOT.BTagEntry.FLAV_C, 'mujets')
btagReader.load(btagData, ROOT.BTagEntry.FLAV_UDSG, 'incl')
btv_sf = makesf(btagReader)
for syst in ['central', 'up', 'down']:
csf = coffea_sf.eval(syst, flavor, abseta, pt, discr)
bsf = btv_sf(syst, flavor, abseta, pt, discr)
print(abs(csf - bsf).max())
def prepare_lookups(self):
# Rochester correction
rochester_data = txt_converters.convert_rochester_file(
self.parameters["roccor_file"], loaduncs=True)
self.roccor_lookup = rochester_lookup.rochester_lookup(rochester_data)
# JEC, JER and uncertainties
self.jec_factories, self.jec_factories_data = jec_factories(self.year)
# Muon scale factors
self.musf_lookup = musf_lookup(self.parameters)
# Pile-up reweighting
self.pu_lookups = pu_lookups(self.parameters)
# Btag weights
self.btag_lookup = BTagScaleFactor(
self.parameters["btag_sf_csv"],
BTagScaleFactor.RESHAPE,
"iterativefit,iterativefit,iterativefit",
)
# STXS VBF cross-section uncertainty
self.stxs_acc_lookups, self.powheg_xsec_lookup = stxs_lookups()
# --- Evaluator
self.extractor = extractor()
# Z-pT reweigting (disabled)
zpt_filename = self.parameters["zpt_weights_file"]
self.extractor.add_weight_sets([f"* * {zpt_filename}"])
if "2016" in self.year:
self.zpt_path = "zpt_weights/2016_value"
else:
self.zpt_path = "zpt_weights/2017_value"
# PU ID weights
puid_filename = self.parameters["puid_sf_file"]
self.extractor.add_weight_sets([f"* * {puid_filename}"])
# Calibration of event-by-event mass resolution
for mode in ["Data", "MC"]:
label = f"res_calib_{mode}_{self.year}"
path = self.parameters["res_calib_path"]
file_path = f"{path}/{label}.root"
self.extractor.add_weight_sets([f"{label} {label} {file_path}"])
self.extractor.finalize()
self.evaluator = self.extractor.make_evaluator()
self.evaluator[self.zpt_path]._axes = self.evaluator[
self.zpt_path]._axes[0]
return
#Optionally, load the dataset from an uncompressed format
else:
print(files_in_batch)
print("loading dataset from cache")
dataset.from_cache(verbose=True, nthreads=args.nthreads)
if not args.cache_only:
if is_mc:
# add information needed for MC corrections
parameters["pu_corrections_target"] = load_puhist_target(
parameters["pu_corrections_file"])
parameters["btag_SF_target"] = BTagScaleFactor(
parameters["btag_SF_{}".format(
parameters["btagging algorithm"])],
BTagScaleFactor.RESHAPE,
'iterativefit,iterativefit,iterativefit',
keep_df=True)
ext = extractor()
for corr in parameters["corrections"]:
ext.add_weight_sets([corr])
ext.finalize()
evaluator = ext.make_evaluator()
if ibatch == 0:
print(dataset.printout())
# in case of DNN evaluation: load model
model = None
if args.DNN:
class btag_scalefactor:
def __init__(self, year, UL=True):
self.year = year
if self.year == 2016:
pass
elif self.year == 2017:
SF_file = os.path.expandvars('$TWHOME/data/btag/DeepJet_106XUL17SF_WPonly_V2.csv')
self.btag_sf = BTagScaleFactor(SF_file, "medium", keep_df=False)
# and load the efficiencies
self.effs = {
'b': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Summer20UL17_b_eff_deepJet.json")),
'c': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Summer20UL17_c_eff_deepJet.json")),
'light': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Summer20UL17_light_eff_deepJet.json")),
}
elif self.year == 2018 and UL:
SF_file = os.path.expandvars('$TWHOME/data/btag/DeepJet_106XUL18SF_WPonly.csv')
self.btag_sf = BTagScaleFactor(SF_file, "medium", keep_df=False)
# and load the efficiencies
self.effs = {
'b': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Summer20UL18_b_eff_deepJet.json")),
'c': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Summer20UL18_c_eff_deepJet.json")),
'light': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Summer20UL18_light_eff_deepJet.json")),
}
elif self.year == 2018 and not UL:
SF_file = os.path.expandvars('$TWHOME/data/btag/DeepJet_102XSF_V2.csv')
self.btag_sf = BTagScaleFactor(SF_file, "medium", keep_df=False)
# and load the efficiencies
self.effs = {
'b': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Autumn18_b_eff_deepJet.json")),
'c': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Autumn18_c_eff_deepJet.json")),
'light': Hist1D.from_json(os.path.expandvars("$TWHOME/data/btag/Autumn18_light_eff_deepJet.json")),
}
def Method1a(self, tagged, untagged, b_direction='central', c_direction='central'):
import numpy as np
'''
tagged: jet collection of tagged jets
untagged: jet collection untagged jets
effs: dictionary of the tagging efficiencies (1D yahist objects)
btag_sf: coffea b-tag SF object
'''
tagged_b = yahist_1D_lookup(self.effs['b'], tagged.pt)*(tagged.hadronFlavour==5)
tagged_c = yahist_1D_lookup(self.effs['c'], tagged.pt)*(tagged.hadronFlavour==4)
tagged_light = yahist_1D_lookup(self.effs['light'], tagged.pt)*(tagged.hadronFlavour==0)
tagged_SFs_b = self.btag_sf.eval(b_direction, tagged.hadronFlavour, abs(tagged.eta), tagged.pt )
tagged_SFs_c = self.btag_sf.eval(c_direction, tagged.hadronFlavour, abs(tagged.eta), tagged.pt )
tagged_SFs_light = self.btag_sf.eval(c_direction, tagged.hadronFlavour, abs(tagged.eta), tagged.pt )
SFs_c = ((tagged_c/tagged_c)*tagged_SFs_c)
SFs_b = ((tagged_b/tagged_b)*tagged_SFs_b)
SFs_light = ((tagged_light/tagged_light)*tagged_SFs_light)
SFs_c = np.where(np.isnan(SFs_c), 0, SFs_c)
SFs_b = np.where(np.isnan(SFs_b), 0, SFs_b)
SFs_light = np.where(np.isnan(SFs_light), 0, SFs_light)
tagged_SFs = SFs_b+SFs_c+SFs_light
untagged_b = yahist_1D_lookup(self.effs['b'], untagged.pt)*(untagged.hadronFlavour==5)
untagged_c = yahist_1D_lookup(self.effs['c'], untagged.pt)*(untagged.hadronFlavour==4)
untagged_light = yahist_1D_lookup(self.effs['light'], untagged.pt)*(untagged.hadronFlavour==0)
untagged_SFs_b = self.btag_sf.eval(b_direction, untagged.hadronFlavour, abs(untagged.eta), untagged.pt )
untagged_SFs_c = self.btag_sf.eval(c_direction, untagged.hadronFlavour, abs(untagged.eta), untagged.pt )
untagged_SFs_light = self.btag_sf.eval(c_direction, untagged.hadronFlavour, abs(untagged.eta), untagged.pt )
SFs_c = ((untagged_c/untagged_c)*untagged_SFs_c)
SFs_b = ((untagged_b/untagged_b)*untagged_SFs_b)
SFs_light = ((untagged_light/untagged_light)*untagged_SFs_light)
SFs_c = np.where(np.isnan(SFs_c), 0, SFs_c)
SFs_b = np.where(np.isnan(SFs_b), 0, SFs_b)
SFs_light = np.where(np.isnan(SFs_light), 0, SFs_light)
untagged_SFs = SFs_b+SFs_c+SFs_light
tagged_all = (tagged_b+tagged_c+tagged_light)
untagged_all = (untagged_b+untagged_c+untagged_light)
denom = ak.prod(tagged_all, axis=1) * ak.prod((1-untagged_all), axis=1)
num = ak.prod(tagged_all*tagged_SFs, axis=1) * ak.prod((1-untagged_all*untagged_SFs), axis=1)
return num/denom
class BTagCorrector:
def __init__(self, year, workingpoint):
self._year = year
self._wp = BTagEfficiency.btagWPs[year][workingpoint]
files = {
'2016': 'DeepCSV_Moriond17_B_H.csv.gz',
'2017': 'DeepCSV_94XSF_V5_B_F.csv.gz',
'2018': 'DeepCSV_102XSF_V1.csv.gz',
}
filename = os.path.join(os.path.dirname(__file__), 'data', files[year])
self.sf = BTagScaleFactor(filename, workingpoint)
files = {
'2016': 'btagQCD2017.coffea',
'2017': 'btagQCD2017.coffea',
'2018': 'btagQCD2017.coffea',
}
filename = os.path.join(os.path.dirname(__file__), 'data', files[year])
btag = util.load(filename)
bpass = btag.integrate('btag', 'pass').values()[()]
ball = btag.integrate('btag').values()[()]
nom = bpass / numpy.maximum(ball, 1.)
dn, up = hist.clopper_pearson_interval(bpass, ball)
self.eff = dense_lookup(nom, [ax.edges() for ax in btag.axes()[1:]])
self.eff_statUp = dense_lookup(up,
[ax.edges() for ax in btag.axes()[1:]])
self.eff_statDn = dense_lookup(dn,
[ax.edges() for ax in btag.axes()[1:]])
def addBtagWeight(self, weights, jets):
abseta = abs(jets.eta)
passbtag = jets.btagDeepB > self._wp
# https://twiki.cern.ch/twiki/bin/viewauth/CMS/BTagSFMethods#1a_Event_reweighting_using_scale
def combine(eff, sf):
# tagged SF = SF*eff / eff = SF
tagged_sf = sf[passbtag].prod()
# untagged SF = (1 - SF*eff) / (1 - eff)
untagged_sf = ((1 - sf * eff) / (1 - eff))[~passbtag].prod()
return tagged_sf * untagged_sf
eff_nom = self.eff(jets.hadronFlavour, jets.pt, abseta)
eff_statUp = self.eff_statUp(jets.hadronFlavour, jets.pt, abseta)
eff_statDn = self.eff_statDn(jets.hadronFlavour, jets.pt, abseta)
sf_nom = self.sf.eval('central', jets.hadronFlavour, abseta, jets.pt)
sf_systUp = self.sf.eval('up', jets.hadronFlavour, abseta, jets.pt)
sf_systDn = self.sf.eval('down', jets.hadronFlavour, abseta, jets.pt)
nom = combine(eff_nom, sf_nom)
weights.add('btagWeight',
nom,
weightUp=combine(eff_nom, sf_systUp),
weightDown=combine(eff_nom, sf_systDn))
weights.add('btagEffStat',
numpy.ones_like(nom),
weightUp=combine(eff_statUp, sf_nom) / nom,
weightDown=combine(eff_statDn, sf_nom) / nom)
for i in numpy.where((nom < 0.01) | (nom > 10)
| numpy.isnan(nom))[0][:4]:
jet = jets[i]
logger.info("Strange weight for event: %r", nom[i])
logger.info(" jet pts: %r", jet.pt)
logger.info(" jet etas: %r", jet.eta)
logger.info(" jet flavors: %r", jet.hadronFlavour)
logger.info(" jet btags: %r", jet.btagDeepB)
logger.info(" result eff: %r up %r down %r", eff_nom[i],
eff_statUp[i], eff_statDn[i])
logger.info(" result sf: %r", sf_nom[i])
return nom
def __init__(self, args, do_tensorflow=True, gpu_memory_fraction=0.2):
"""Summary
Args:
args (Namespace): Command line arguments from analysis_hmumu.parse_args
do_tensorflow (bool, optional): If True, enable the use of tensorflow-based DNN evaluation
gpu_memory_fraction (float, optional): What fraction of GPU memory to allocate to tensorflow
"""
self.lumimask = {
"2016":
LumiMask(
"data/Cert_271036-284044_13TeV_ReReco_07Aug2017_Collisions16_JSON.txt",
np, backend_cpu),
"2017":
LumiMask(
"data/Cert_294927-306462_13TeV_EOY2017ReReco_Collisions17_JSON_v1.txt",
np, backend_cpu),
"2018":
LumiMask(
"data/Cert_314472-325175_13TeV_17SeptEarlyReReco2018ABC_PromptEraD_Collisions18_JSON.txt",
np, backend_cpu),
}
self.lumidata = {
"2016": LumiData("data/lumi2016.csv"),
"2017": LumiData("data/lumi2017.csv"),
"2018": LumiData("data/lumi2018.csv")
}
self.pu_corrections = {
"2016": load_puhist_target("data/pileup/RunII_2016_data.root"),
"2017": load_puhist_target("data/pileup/RunII_2017_data.root"),
"2018": load_puhist_target("data/pileup/RunII_2018_data.root")
}
#Load the C++ helper library
try:
self.libhmm = LibHMuMu()
except OSError as e:
print(e, file=sys.stderr)
print(
"Could not load the C++ helper library, please make sure it's compiled by doing",
file=sys.stderr)
print(" $ cd tests/hmm && make -j4", file=sys.stderr)
print(
"If there are still issues, please make sure ROOT is installed"
)
print(
"and the library is compiled against the same version as seen at runtime",
file=sys.stderr)
sys.exit(1)
print("Loading Rochester corrections")
#https://twiki.cern.ch/twiki/bin/viewauth/CMS/RochcorMuon
self.rochester_corrections = {
"2016": RochesterCorrections(self.libhmm, "data/RoccoR2016.txt"),
"2017": RochesterCorrections(self.libhmm, "data/RoccoR2017.txt"),
"2018": RochesterCorrections(self.libhmm, "data/RoccoR2018.txt")
}
#Luminosity weight ratios for computing lepton scale factors
self.ratios_dataera = {
#BCDEF, GH
"2016": [0.5548, 1.0 - 0.5548],
"2017": 1.0,
"2018": 1.0
}
print("Loading lepton SF")
self.lepsf_iso = {
"2016":
LeptonEfficiencyCorrections(self.libhmm, [
"data/leptonSF/2016/RunBCDEF_SF_ISO.root",
"data/leptonSF/2016/RunGH_SF_ISO.root"
], [
"NUM_LooseRelIso_DEN_MediumID_eta_pt",
"NUM_LooseRelIso_DEN_MediumID_eta_pt"
], self.ratios_dataera["2016"]),
"2017":
LeptonEfficiencyCorrections(
self.libhmm, ["data/leptonSF/2017/RunBCDEF_SF_ISO_syst.root"],
["NUM_LooseRelIso_DEN_MediumID_pt_abseta"], [1.0]),
"2018":
LeptonEfficiencyCorrections(
self.libhmm, ["data/leptonSF/2018/RunABCD_SF_ISO.root"],
["NUM_LooseRelIso_DEN_MediumID_pt_abseta"], [1.0]),
}
self.lepsf_id = {
"2016":
LeptonEfficiencyCorrections(self.libhmm, [
"data/leptonSF/2016/RunBCDEF_SF_ID.root",
"data/leptonSF/2016/RunGH_SF_ID.root"
], [
"NUM_MediumID_DEN_genTracks_eta_pt",
"NUM_MediumID_DEN_genTracks_eta_pt"
], self.ratios_dataera["2016"]),
"2017":
LeptonEfficiencyCorrections(
self.libhmm, ["data/leptonSF/2017/RunBCDEF_SF_ID_syst.root"],
["NUM_MediumID_DEN_genTracks_pt_abseta"], [1.0]),
"2018":
LeptonEfficiencyCorrections(
self.libhmm, ["data/leptonSF/2018/RunABCD_SF_ID.root"],
["NUM_MediumID_DEN_TrackerMuons_pt_abseta"], [1.0])
}
self.lepeff_trig_data = {
"2016":
LeptonEfficiencyCorrections(self.libhmm, [
"data/leptonSF/2016/EfficienciesAndSF_RunBtoF.root",
"data/leptonSF/2016/EfficienciesAndSF_RunGtoH.root"
], [
"IsoMu24_OR_IsoTkMu24_PtEtaBins/efficienciesDATA/abseta_pt_DATA",
"IsoMu24_OR_IsoTkMu24_PtEtaBins/efficienciesDATA/abseta_pt_DATA"
], self.ratios_dataera["2016"]),
"2017":
LeptonEfficiencyCorrections(self.libhmm, [
"data/leptonSF/2017/EfficienciesAndSF_RunBtoF_Nov17Nov2017.root"
], ["IsoMu27_PtEtaBins/efficienciesDATA/pt_abseta_DATA"], [1.0]),
"2018":
LeptonEfficiencyCorrections(self.libhmm, [
"data/leptonSF/2018/EfficienciesAndSF_2018Data_AfterMuonHLTUpdate.root"
], ["IsoMu24_PtEtaBins/efficienciesDATA/pt_abseta_DATA"], [1.0]),
}
self.lepeff_trig_mc = {
"2016":
LeptonEfficiencyCorrections(self.libhmm, [
"data/leptonSF/2016/EfficienciesAndSF_RunBtoF.root",
"data/leptonSF/2016/EfficienciesAndSF_RunGtoH.root"
], [
"IsoMu24_OR_IsoTkMu24_PtEtaBins/efficienciesMC/abseta_pt_MC",
"IsoMu24_OR_IsoTkMu24_PtEtaBins/efficienciesMC/abseta_pt_MC"
], self.ratios_dataera["2016"]),
"2017":
LeptonEfficiencyCorrections(self.libhmm, [
"data/leptonSF/2017/EfficienciesAndSF_RunBtoF_Nov17Nov2017.root"
], ["IsoMu27_PtEtaBins/efficienciesMC/pt_abseta_MC"], [1.0]),
"2018":
LeptonEfficiencyCorrections(self.libhmm, [
"data/leptonSF/2018/EfficienciesAndSF_2018Data_AfterMuonHLTUpdate.root"
], ["IsoMu24_PtEtaBins/efficienciesMC/pt_abseta_MC"], [1.0]),
}
print("Loading JEC...")
#JEC files copied from
#https://github.com/cms-jet/JECDatabase/tree/master/textFiles
#Need to rename JECDatabase files as follows:
# *_UncertaintySources_AK4PFchs.txt -> *_UncertaintySources_AK4PFchs.junc.txt
# *_Uncertainty_AK4PFchs.txt -> *_Uncertainty_AK4PFchs.junc.txt
#JER files from
# *_PtResolution_AK4PFchs.txt -> *_PtResolution_AK4PFchs.jr.txt
# *_SF_AK4PFchs.txt -> *_SF_AK4PFchs.jersf.txt
self.jetmet_corrections = {
"2016": {
"Summer16_07Aug2017_V11":
JetMetCorrections(jec_tag="Summer16_07Aug2017_V11_MC",
jec_tag_data={
"RunB": "Summer16_07Aug2017BCD_V11_DATA",
"RunC": "Summer16_07Aug2017BCD_V11_DATA",
"RunD": "Summer16_07Aug2017BCD_V11_DATA",
"RunE": "Summer16_07Aug2017EF_V11_DATA",
"RunF": "Summer16_07Aug2017EF_V11_DATA",
"RunG": "Summer16_07Aug2017GH_V11_DATA",
"RunH": "Summer16_07Aug2017GH_V11_DATA",
},
jer_tag="Summer16_25nsV1_MC",
do_factorized_jec=True),
},
"2017": {
"Fall17_17Nov2017_V32":
JetMetCorrections(jec_tag="Fall17_17Nov2017_V32_MC",
jec_tag_data={
"RunB": "Fall17_17Nov2017B_V32_DATA",
"RunC": "Fall17_17Nov2017C_V32_DATA",
"RunD": "Fall17_17Nov2017DE_V32_DATA",
"RunE": "Fall17_17Nov2017DE_V32_DATA",
"RunF": "Fall17_17Nov2017F_V32_DATA",
},
jer_tag="Fall17_V3_MC",
do_factorized_jec=True),
},
"2018": {
"Autumn18_V19":
JetMetCorrections(jec_tag="Autumn18_V19_MC",
jec_tag_data={
"RunA": "Autumn18_RunA_V19_DATA",
"RunB": "Autumn18_RunB_V19_DATA",
"RunC": "Autumn18_RunC_V19_DATA",
"RunD": "Autumn18_RunD_V19_DATA",
},
jer_tag="Autumn18_V7_MC",
do_factorized_jec=True),
}
}
self.dnn_model = None
self.dnn_normfactors = None
self.dnnPisa_models = []
self.dnnPisa_normfactors1 = None
self.dnnPisa_normfactors2 = None
if do_tensorflow:
print("Loading tensorflow model")
#disable GPU for tensorflow
import tensorflow as tf
config = tf.compat.v1.ConfigProto()
config.intra_op_parallelism_threads = 1
config.inter_op_parallelism_threads = 1
if not args.use_cuda:
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
else:
config.gpu_options.allow_growth = False
config.gpu_options.per_process_gpu_memory_fraction = gpu_memory_fraction
tf.compat.v1.keras.backend.set_session(
tf.compat.v1.Session(config=config))
#load DNN model
import keras
self.dnn_model = keras.models.load_model(
"data/DNN27vars_sig_vbf_ggh_bkg_dyvbf_dy105To160_ewk105To160_split_60_40_mod10_200109.h5"
)
self.dnn_normfactors = np.load(
"data/DNN27vars_sig_vbf_ggh_bkg_dyvbf_dy105To160_ewk105To160_split_60_40_mod10_200109.npy"
)
if args.use_cuda:
import cupy
self.dnn_normfactors = cupy.array(
self.dnn_normfactors[0]), cupy.array(
self.dnn_normfactors[1])
for imodel in range(4):
dnnPisa_model = keras.models.load_model(
"data/PisaDNN/model_preparation/nn_evt" + str(imodel) +
"_february.h5")
self.dnnPisa_models += [dnnPisa_model]
self.dnnPisa_normfactors1 = np.load(
"data/PisaDNN/model_preparation/Febhelp_node1.npy")
self.dnnPisa_normfactors2 = np.load(
"data/PisaDNN/model_preparation/Febhelp_node2.npy")
if args.use_cuda:
import cupy
self.dnnPisa_normfactors1 = cupy.array(
self.dnnPisa_normfactors1[0]), cupy.array(
self.dnnPisa_normfactors1[1])
self.dnnPisa_normfactors2 = cupy.array(
self.dnnPisa_normfactors2[0]), cupy.array(
self.dnnPisa_normfactors2[1])
print("Loading UCSD BDT model")
#self.bdt_ucsd = GBREvaluator(self.libhmm, "data/Hmm_BDT_xml/2016/TMVAClassification_BDTG.weights.2jet_bveto_withmass.xml")
#self.bdt2j_ucsd = {
# "2016": GBREvaluator(self.libhmm, "data/Hmm_BDT_xml/2016/TMVAClassification_BDTG.weights.2jet_bveto.xml"),
# "2017": GBREvaluator(self.libhmm, "data/Hmm_BDT_xml/2017/TMVAClassification_BDTG.weights.2jet_bveto.xml"),
# "2018": GBREvaluator(self.libhmm, "data/Hmm_BDT_xml/2018/TMVAClassification_BDTG.weights.2jet_bveto.xml")
#}
#self.bdt01j_ucsd = {
# "2016": GBREvaluator(self.libhmm, "data/Hmm_BDT_xml/2016/TMVAClassification_BDTG.weights.01jet.xml"),
# "2017": GBREvaluator(self.libhmm, "data/Hmm_BDT_xml/2017/TMVAClassification_BDTG.weights.01jet.xml"),
# "2018": GBREvaluator(self.libhmm, "data/Hmm_BDT_xml/2018/TMVAClassification_BDTG.weights.01jet.xml")
#}
self.miscvariables = MiscVariables(self.libhmm)
print("Loading NNLOPSReweighting...")
self.nnlopsreweighting = NNLOPSReweighting(
self.libhmm, "data/nnlops/NNLOPS_reweight.root")
print("Loading hRelResolution...")
self.hrelresolution = hRelResolution(
self.libhmm, "data/PisaDNN/muonresolution.root")
print("Loading ZpTReweighting...")
self.zptreweighting = ZpTReweighting(self.libhmm)
puid_maps = "data/puidSF/PUIDMaps.root"
print("Extracting PU ID weights from " + puid_maps)
puid_extractor = extractor()
puid_extractor.add_weight_sets(["* * {0}".format(puid_maps)])
puid_extractor.finalize()
self.puidreweighting = puid_extractor.make_evaluator()
print("Extracting b-tag weights...")
self.btag_weights = {
"DeepCSV_2016":
BTagScaleFactor("data/btagSF/DeepCSV_2016LegacySF_V1.csv",
BTagScaleFactor.RESHAPE,
'iterativefit,iterativefit,iterativefit',
keep_df=True),
"DeepCSV_2017":
BTagScaleFactor("data/btagSF/DeepCSV_94XSF_V5_B_F.csv",
BTagScaleFactor.RESHAPE,
'iterativefit,iterativefit,iterativefit',
keep_df=True),
"DeepCSV_2018":
BTagScaleFactor("data/btagSF/DeepCSV_102XSF_V1.csv",
BTagScaleFactor.RESHAPE,
'iterativefit,iterativefit,iterativefit',
keep_df=True)
#"DeepCSV_2016": BTagWeights(tag_name="DeepCSV_2016LegacySF_V1"),
#"DeepCSV_2017": BTagWeights(tag_name="DeepCSV_94XSF_V4_B_F"),
#"DeepCSV_2018": BTagWeights(tag_name="DeepCSV_102XSF_V1")
}
from coffea.btag_tools import BTagScaleFactor
btag_sf = BTagScaleFactor("data/DeepCSV_2016LegacySF_V1.csv.gz", "medium")
## ele tight ID weights
from coffea.lookup_tools import extractor
ext = extractor()
ext.add_weight_sets([
"EGamma_SF2D_T EGamma_SF2D data/2016LegacyReReco_ElectronTight_Fall17V2.root",
"EGamma_SF2D_T_err EGamma_SF2D_error data/2016LegacyReReco_ElectronTight_Fall17V2.root"
])
ext.add_weight_sets([
"EGamma_SF2D_L EGamma_SF2D data/2016LegacyReReco_ElectronLoose_Fall17V2.root",
"EGamma_SF2D_L_err EGamma_SF2D_error data/2016LegacyReReco_ElectronLoose_Fall17V2.root"
])
ext.add_weight_sets([
"EGamma_SF2D_Trig hEffEtaPt data/electron_Trigger_eleTrig.root",
"EGamma_SF2D_Trig_err hEffEtaPt_error data/electron_Trigger_eleTrig.root"
])
ext.add_weight_sets([
"EGamma_SF2D_Reco EGamma_SF2D data/EGM2D_BtoH_GT20GeV_RecoSF_Legacy2016.root",
"EGamma_SF2D_Reco_err EGamma_SF2D_error data/EGM2D_BtoH_GT20GeV_RecoSF_Legacy2016.root"
])
ext.add_weight_sets([
"EGamma_SF2D_Reco_lowpt EGamma_SF2D data/EGM2D_BtoH_low_RecoSF_Legacy2016.root",
"EGamma_SF2D_Reco_lowpt_err EGamma_SF2D_error data/EGM2D_BtoH_low_RecoSF_Legacy2016.root"
])
def test_BTagScalefactor():
sf1 = BTagScaleFactor('tests/samples/testBTagSF.btag.csv', 'medium')
sf2 = BTagScaleFactor('tests/samples/DeepCSV_102XSF_V1.btag.csv.gz',
BTagScaleFactor.RESHAPE, 'iterativefit')
sf3 = BTagScaleFactor('tests/samples/DeepCSV_102XSF_V1.btag.csv.gz',
BTagScaleFactor.TIGHT)
sf4 = BTagScaleFactor(
'tests/samples/DeepCSV_2016LegacySF_V1_TuneCP5.btag.csv.gz',
BTagScaleFactor.RESHAPE,
'iterativefit',
keep_df=True)
# import pdb; pdb.set_trace()
counts, test_eta, test_pt = dummy_jagged_eta_pt()
test_flavor = numpy.random.choice([0, 4, 5], size=len(test_eta))
test_allb = numpy.ones_like(test_flavor) * 5
test_discr = numpy.random.rand(len(test_eta))
offsets = numpy.zeros(len(counts) + 1)
offsets[1:] = numpy.cumsum(counts)
test_jets = ak.Array(
ak.layout.ListOffsetArray64(
ak.layout.Index64(offsets),
ak.zip(
{
"pt": test_pt,
"eta": test_eta,
"flavor": test_flavor,
"btag": test_discr,
},
highlevel=False)))
expected = numpy.array([
0.93724101, 0.89943609, 1.0671185, 1.06846618, 0.94530984, 1.06645614,
0.91862676, 1.06645614, 0.94372127, 0.94505261, 1.06645614, 1.06645614,
1.06645614, 1.06645614, 0.91385676, 1.06738093, 0.89943609, 0.92593492,
1.06960044, 0.89943609, 1.06645614, 1.06645614, 0.94290361, 1.06892548,
0.92440686, 0.92046542, 1.06645614, 0.93676041, 0.93392431, 0.91694353,
0.89943609, 0.89943609, 0.89943609, 0.89943609, 0.89943609, 0.89943609,
0.89943609, 0.89943609, 0.93371251, 0.89943609, 0.89943609, 0.89943609,
0.94767034, 1.06645614, 1.0670672, 1.07136352, 0.89943609, 0.90445481,
0.89943609, 1.06645614, 0.89943609, 0.89943609, 0.93745389, 0.90949125,
0.91778825, 1.06645614, 1.06645614, 0.89943609, 0.89943609, 1.06645614,
1.06645614, 1.06645614
])
result = sf1.eval('central', test_flavor, test_eta, test_pt, test_discr)
assert numpy.allclose(result, expected)
sf1.eval('up', test_flavor, test_eta, test_pt)
sf2.eval('central', test_allb, test_eta, test_pt, test_discr)
with pytest.raises(ValueError):
sf2.eval('up', test_allb, test_eta, test_pt)
sf3.eval('central', test_flavor, test_eta, test_pt, test_discr)
sf3.eval('up', test_flavor, test_eta, test_pt)
with pytest.raises(ValueError):
sf4.eval('central', test_flavor, test_eta, test_pt, test_discr)
expected = numpy.array([
1.2185781, 1.03526095, 1.14997077, 0.91933821, 1.2185781, 1.08865945,
0.99422718, 1.01943199, 1.01025089, 1.20312875, 0.84198391, 0.91726759,
0.93501452, 1.31649974, 1.14997077, 1.02107876, 1.06150099, 1.06063444,
0.90508972, 1.20768481, 0.8484613, 0.99217259, 0.98333802, 1.31302575,
1.0104926, 1.00474285, 1.24375693, 1.20949677, 0.91714979, 0.99533782,
1.14997077, 1.02871797, 0.99619147, 0.97543142, 1.31518787, 1.30700837,
1.14997077, 0.99879282, 0.98961045, 1.14997077, 0.88343516, 0.9930647,
1.17767042, 1.14997077, 1.30594256, 0.91888068, 1.04737201, 1.03583147,
1.02833176, 0.99527427, 0.98546895, 1.14997077, 1.04815223, 1.28007547,
1.1970858, 1.12892238, 1.14997077, 1.14997077, 1.01656481, 0.84198391,
1.2996388, 1.14997077
])
result = sf4.eval('central', test_allb, test_eta, test_pt, test_discr)
assert numpy.allclose(result, expected)
sf1.eval('down', test_jets.flavor, test_jets.eta, test_jets.pt)
fi = uproot.open("data/myNanoProdMc2016_NANO_skim.root")
tt = fi.get("Events")
arr_flav = np.array(tt.array("Jet_hadronFlavour").content, np.int32)
arr_abs_eta = np.array(np.abs(tt.array("Jet_eta").content), np.float32)
arr_pt = np.array(tt.array("Jet_pt").content, np.float32)
arr_discr = np.array(tt.array("Jet_btagDeepB").content, np.float32)
systs = [
"jes", "lfstats1", "lfstats2", "hfstats1", "hfstats2", "cferr1",
"cferr2", "lf", "hf"
]
#BTagScaleFactor from coffea
t0 = time.time()
sf = BTagScaleFactor(sf_file,
BTagScaleFactor.RESHAPE,
'iterativefit,iterativefit,iterativefit',
keep_df=True)
for tsys in systs:
for sdir in ["up", "down"]:
tsys_name = sdir + '_' + tsys
sf.eval(tsys_name, arr_flav[:1], arr_abs_eta[:1], arr_pt[:1],
arr_discr[:1], True)
t1 = time.time()
print("init_py", t1 - t0)
#BTagCalibrationStandalone from POG
t0 = time.time()
libhmm = LibHMuMu()
print("loading BTagCalibration")
systs_sdir = []
for sdir in ["up", "down"]:
class btag_scalefactor:
def __init__(self, year):
self.year = year
if self.year == 2016:
pass
elif self.year == 2017:
pass
elif self.year == 2018:
SF_file = os.path.expandvars(
'$TWHOME/data/btag/DeepJet_102XSF_V2.csv')
self.btag_sf = BTagScaleFactor(SF_file, "medium", keep_df=False)
# and load the efficiencies
self.effs = {
'b':
Hist1D.from_json(
os.path.expandvars(
"$TWHOME/data/btag/Autumn18_b_eff_deepJet.json")),
'c':
Hist1D.from_json(
os.path.expandvars(
"$TWHOME/data/btag/Autumn18_c_eff_deepJet.json")),
'light':
Hist1D.from_json(
os.path.expandvars(
"$TWHOME/data/btag/Autumn18_light_eff_deepJet.json")),
}
def Method1a(self, tagged, untagged):
'''
tagged: jet collection of tagged jets
untagged: jet collection untagged jets
effs: dictionary of the tagging efficiencies (1D yahist objects)
btag_sf: coffea b-tag SF object
'''
tagged_b = yahist_1D_lookup(self.effs['b'],
tagged.pt) * (tagged.hadronFlavour == 5)
tagged_c = yahist_1D_lookup(self.effs['c'],
tagged.pt) * (tagged.hadronFlavour == 4)
tagged_light = yahist_1D_lookup(
self.effs['light'], tagged.pt) * (tagged.hadronFlavour == 0)
tagged_SFs = self.btag_sf.eval('central', tagged.hadronFlavour,
abs(tagged.eta), tagged.pt)
untagged_b = yahist_1D_lookup(
self.effs['b'], untagged.pt) * (untagged.hadronFlavour == 5)
untagged_c = yahist_1D_lookup(
self.effs['c'], untagged.pt) * (untagged.hadronFlavour == 4)
untagged_light = yahist_1D_lookup(
self.effs['light'], untagged.pt) * (untagged.hadronFlavour == 0)
untagged_SFs = self.btag_sf.eval('central', untagged.hadronFlavour,
abs(untagged.eta), untagged.pt)
tagged_all = (tagged_b + tagged_c + tagged_light)
untagged_all = (untagged_b + untagged_c + untagged_light)
denom = ak.prod(tagged_all, axis=1) * ak.prod(
(1 - untagged_all), axis=1)
num = ak.prod(tagged_all * tagged_SFs, axis=1) * ak.prod(
(1 - untagged_all * untagged_SFs), axis=1)
return num / denom
out_events["mask_topenu2b"] = cms_events["mask_topenu2b"]
out_events["mask_topenu1b"] = cms_events["mask_topenu1b"]
out_events["mask_topmunu2b"] = cms_events["mask_topmunu2b"]
out_events["mask_topmunu1b"] = cms_events["mask_topmunu1b"]
out_events["mask_wenu1b"] = cms_events["mask_wenu1b"]
out_events["mask_wenu2b"] = cms_events["mask_wenu2b"]
out_events["mask_wmunu1b"] = cms_events["mask_wmunu1b"]
out_events["mask_wmunu2b"] = cms_events["mask_wmunu2b"]
#### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####
#### #### the following code to read the SFs can be moved in another python file #### #### ####
#### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####
## btagging SFs
from coffea.btag_tools import BTagScaleFactor
btag_sf = BTagScaleFactor("data/DeepCSV_2016LegacySF_V1.csv.gz", "medium")
out_events["btagsf0"] = btag_sf.eval("central", out_events.jetflav0,
abs(out_events.jeteta0),
out_events.jetpt0)
out_events["btagsf1"] = btag_sf.eval("central", out_events.jetflav1,
abs(out_events.jeteta1),
out_events.jetpt1)
out_events["btagsf2"] = btag_sf.eval("central", out_events.jetflav2,
abs(out_events.jeteta2),
out_events.jetpt2)
out_events["btagsf3"] = btag_sf.eval("central", out_events.jetflav3,
abs(out_events.jeteta3),
out_events.jetpt3)
out_events["btagsf4"] = btag_sf.eval("central", out_events.jetflav4,
abs(out_events.jeteta4),
out_events.jetpt4)
