def get_hlt_prescales(runs):
fn = 'prescales_temp/hltprescales.gzpickle'
if os.path.isfile(fn):
return from_pickle(fn)
prescales = defaultdict(lambda: defaultdict(lambda: defaultdict(set)))
l = len(runs)
for i,run in enumerate(runs):
print 'get_hlt_prescales run %i (%i/%i)' % (run, i+1, l)
res = lumi_context('hltbyls', {run:None})[run]
for line in res:
ls = line[0]
for hlt_name, prescale, duh, duh2 in line[-1]:
prescales[hlt_name][prescale][run].add(ls)
for hlt_name, duh in prescales.iteritems():
for prescale, duh2 in duh.iteritems():
for run in duh2:
duh2[run] = collapse(duh2[run])
duh[prescale] = dict(duh2)
prescales[hlt_name] = dict(duh)
prescales = dict(prescales)
to_pickle(prescales, fn)
return prescales
def strip(cls, fn, new_fn):
lls = from_pickle(fn, comp=True)
for ll in lls:
del ll.time
del ll.beam_status
del ll.energy
to_pickle(lls, new_fn, comp=True)
def get_hlt_prescales(runs):
fn = 'prescales_temp/hltprescales.gzpickle'
if os.path.isfile(fn):
return from_pickle(fn)
prescales = defaultdict(lambda: defaultdict(lambda: defaultdict(set)))
l = len(runs)
for i, run in enumerate(runs):
print 'get_hlt_prescales run %i (%i/%i)' % (run, i + 1, l)
res = lumi_context('hltbyls', {run: None})[run]
for line in res:
ls = line[0]
for hlt_name, prescale, duh, duh2 in line[-1]:
prescales[hlt_name][prescale][run].add(ls)
for hlt_name, duh in prescales.iteritems():
for prescale, duh2 in duh.iteritems():
for run in duh2:
duh2[run] = collapse(duh2[run])
duh[prescale] = dict(duh2)
prescales[hlt_name] = dict(duh)
prescales = dict(prescales)
to_pickle(prescales, fn)
return prescales
def strip(cls, fn, new_fn):
lls = from_pickle(fn, comp=True)
for ll in lls:
del ll.time
del ll.beam_status
del ll.energy
to_pickle(lls, new_fn, comp=True)
def analyze(plots=False):
if plots:
set_style()
ps = plot_saver(plot_dir('pdfweights', temp=True), size=(600, 600))
fields = ('nominal', 'plus', 'minus', 'pdfunc', 'rel', 'relavg', 'n16',
'n84', 'ctl', 'ctlrel', 'ctlrelmd')
fmt = '%-45s' + ' %10s' * len(fields)
print fmt % (('sname', ) + fields)
ss = samples()
for s in ss:
f = ROOT.TFile.Open(s.fn)
s.norm = norm_from_file(f)
s.w = ac.int_lumi_2018 * 1e-3 / s.norm
t = f.Get('mfvMiniTree/t')
hr = draw_hist_register(t, True, True)
if plots:
for i in xrange(48, 148):
h = hr.draw('weights[%i]' % i, 'nvtx>=2', binning='100,0,2')
ps.save(str(i))
cut = '(nvtx>=2)' # '(nvtx>=2 && (gen_x[0]**2 + gen_y[0]**2)**0.5<0.08 && (gen_x[1]**2+gen_y[1]**2)**0.5 < 0.08)'
h = hr.draw('weight', cut, binning='1,0,1')
s.nominal, _ = tuple(s.w * x for x in get_integral(h))
s.pdfunc, s.family = [], []
for i in xrange(
46, 148
): # JMTBAD indices into misc_weights are fragile, need to put the pdf weights/info somewhere else than weights
h = hr.draw('weight',
'%s*misc_weights[%i]' % (cut, i),
binning='1,0,1')
v = tuple(s.w * x for x in get_integral(h))[0]
(s.pdfunc if i < 48 else s.family).append(v)
s.plus, s.minus = s.pdfunc
s.pdfunc = abs(s.plus - s.minus) / 2
s.plus = abs(s.plus - s.nominal)
s.minus = abs(s.nominal - s.minus)
s.rel = s.pdfunc / s.nominal
s.family.sort()
s.n16, s.n84 = s.family[16], s.family[83]
s.ctl = (s.n84 - s.n16) / 2
s.avg = avg(s.family)
s.rms = rms(s.family)
s.mdn = (s.family[49] + s.family[50]) / 2
s.md = (s.n84 + s.n16) / 2
s.relavg = s.pdfunc / s.avg
s.ctlrel = s.ctl / s.nominal
s.ctlrelmd = s.ctl / s.md
print fmt.replace('%10s', '%10.4f') % (
(s.name, ) + tuple(getattr(s, fi) for fi in fields))
to_pickle(ss, 'pdfuncerts.gzpickle')
def get_lumibylses(runs):
fn = 'prescales_temp/lumibylses.gzpickle'
if os.path.isfile(fn):
return from_pickle(fn)
d = {}
for run in runs:
d.update(parse_lumibyls(run))
to_pickle(d, fn)
return d
def get_hlt_prescales_collapsed(runs, hlt_prescales=None):
fn = 'prescales_temp/hltprescales_collapsed.gzpickle'
if os.path.isfile(fn):
return from_pickle(fn)
if hlt_prescales is None:
hlt_prescales = get_hlt_prescales(runs)
hlt_prescales_collapsed = collapse_versions(hlt_prescales)
to_pickle(hlt_prescales_collapsed, fn)
return hlt_prescales_collapsed
def get_lumibylses(runs):
fn = 'prescales_temp/lumibylses.gzpickle'
if os.path.isfile(fn):
return from_pickle(fn)
d = {}
for run in runs:
d.update(parse_lumibyls(run))
to_pickle(d, fn)
return d
def get_hlt_prescales_collapsed(runs, hlt_prescales=None):
fn = 'prescales_temp/hltprescales_collapsed.gzpickle'
if os.path.isfile(fn):
return from_pickle(fn)
if hlt_prescales is None:
hlt_prescales = get_hlt_prescales(runs)
hlt_prescales_collapsed = collapse_versions(hlt_prescales)
to_pickle(hlt_prescales_collapsed, fn)
return hlt_prescales_collapsed
def get_menus(runs):
fn = 'prescales_temp/menus.gzpickle'
if os.path.isfile(fn):
return from_pickle(fn)
menus = defaultdict(set)
for run in runs:
menu = lumi_context('hltmenu', {run:None})[run]
for hlt_path, l1_seed, l1_bit in menu:
assert l1_bit == 'n/a' or l1_seed == l1_bit
menu = tuple([(hlt_path, l1_seed) for hlt_path, l1_seed, l1_bit in menu])
menus[menu].add(run)
to_pickle(menus, fn)
return menus
def get_menus(runs):
fn = 'prescales_temp/menus.gzpickle'
if os.path.isfile(fn):
return from_pickle(fn)
menus = defaultdict(set)
for run in runs:
menu = lumi_context('hltmenu', {run: None})[run]
for hlt_path, l1_seed, l1_bit in menu:
assert l1_bit == 'n/a' or l1_seed == l1_bit
menu = tuple([(hlt_path, l1_seed)
for hlt_path, l1_seed, l1_bit in menu])
menus[menu].add(run)
to_pickle(menus, fn)
return menus
def save(cls, csv_fn, fn):
lls = cls.load_csv(csv_fn)
to_pickle(lls, fn, comp=True)
return lls
import os
try:
print 'getting minbias file list from cache'
from minbias_files import files
except ImportError:
print 'nope, getting minbias file list from DBS and caching'
from JMTucker.Tools.DBS import files_in_dataset
from JMTucker.Tools.general import to_pickle
files = files_in_dataset('/MinBias_TuneCUETP8M1_13TeV-pythia8/RunIISummer15GS-MCRUN2_71_V1-v2/GEN-SIM')
to_pickle(files, 'minbias_files.pkl', comp=True)
open('minbias_files.py', 'wt').write("import cPickle, gzip; files = cPickle.load(gzip.GzipFile('minbias_files.pkl', 'rb'))\n")
def save(cls, csv_fn, fn):
lls = cls.load_csv(csv_fn)
to_pickle(lls, fn, comp=True)
return lls
def analyze(plots=False):
if plots:
set_style()
ps = plot_saver(plot_dir('scaleweights', temp=True), size=(600, 600))
fields = ('nominal', 'n16', 'n84', 'ctl', 'ctlrel', 'ctlrelmd')
fmt = '%-45s' + ' %10s' * len(fields)
print fmt % (('sname', ) + fields)
ss = samples()
for s in ss:
f = ROOT.TFile.Open(s.fn)
s.norm = norm_from_file(f)
s.w = ac.int_lumi_2018 * 1e-3 / s.norm
t = f.Get('mfvMiniTree/t')
hr = draw_hist_register(t, True, True)
weight_str_list = [
"ren_weight_up", "ren_weight_dn", "fac_weight_up", "fac_weight_dn",
"ren_weight_up*fac_weight_up", "ren_weight_dn*fac_weight_dn"
]
cut = '(nvtx>=2)' # '(nvtx>=2 && (gen_x[0]**2 + gen_y[0]**2)**0.5<0.08 && (gen_x[1]**2+gen_y[1]**2)**0.5 < 0.08)'
h = hr.draw('weight', cut, binning='1,0,1')
s.nominal, _ = tuple(s.w * x for x in get_integral(h))
s.scaleunc, s.family = [], []
for weight_str in weight_str_list:
# because I didn't pick better names...
if "ren" in weight_str and "fac" in weight_str:
weight_name_for_norm = "sum_weight_ren_fac"
elif "ren" in weight_str:
weight_name_for_norm = "sum_weight_ren"
elif "fac" in weight_str:
weight_name_for_norm = "sum_weight_fac"
if "_up" in weight_str:
weight_name_for_norm += "_up"
elif "_dn" in weight_str:
weight_name_for_norm += "_dn"
s.norm = norm_from_file_weight(f, weight_name=weight_name_for_norm)
s.w = ac.int_lumi_2018 * 1e-3 / s.norm
h = hr.draw('weight', '%s*%s' % (cut, weight_str), binning='1,0,1')
v = tuple(s.w * x for x in get_integral(h))[0]
#print '%s*%s, weight*nvtx==2 = %s' % (cut,weight_str,v)
s.family.append(v)
if len(weight_str_list) == 6:
s.family.sort()
s.n16, s.n84 = s.family[1], s.family[4] # since total is 6
s.ctl = max(abs(s.family[0] - s.nominal),
abs(s.family[5] -
s.nominal)) # i.e. maximum deviation ==> envelope
#s.ctl = (s.n84 - s.n16)/2
s.avg = avg(s.family)
s.rms = rms(s.family)
s.mdn = (s.family[2] + s.family[3]) / 2
s.md = (s.n84 + s.n16) / 2
#s.relavg = s.scaleunc / s.avg
if s.nominal == 0:
print "s.nominal is 0!!!!!!!!!!!!"
s.nominal = 0.00001
if s.md == 0:
print "s.md is 0!!!!!!!!!!!!"
s.md = 0.00001
s.ctlrel = s.ctl / s.nominal
s.ctlrelmd = s.ctl / s.md
print fmt.replace('%10s', '%10.4f') % (
(s.name, ) + tuple(getattr(s, fi) for fi in fields))
else:
print "invalid weight_str_list!!!!!!!!!!!!!!!!!"
to_pickle(ss, 'ren_fac.gzpickle')
skipped[hlt_path].append(run)
continue
for ls_range in ls_ranges:
l1_prescale = l1_prescale_for_run_ls(
l1_seed, run, ls_range)
total[hlt_path][(hlt_prescale,
l1_prescale)][run].append(ls_range)
for name, duh in total.iteritems():
for prescale, duh2 in duh.iteritems():
duh[prescale] = dict(duh2)
total[name] = dict(duh)
total = dict(total)
if skipped:
print 'warning: skipped these:'
pprint(skipped)
return total
total_prescales = get_total_prescales(hlt_prescales)
total_prescales_by_intlumi = prescales_by_intlumi(total_prescales)
total_prescales_collapsed = collapse_versions(total_prescales)
total_prescales_collapsed_by_intlumi = prescales_by_intlumi(
total_prescales_collapsed)
for x in 'total_prescales total_prescales_by_intlumi total_prescales_collapsed total_prescales_collapsed_by_intlumi'.split(
):
to_pickle(eval(x), 'prescales_temp/' + x + '.gzpickle')
for hlt_prescale, runls in prescales_by_runls.iteritems():
for run, ls_ranges in runls.iteritems():
try:
l1_seed = l1_seed_for_hlt_path_in_run(hlt_path, run)
except ValueError:
skipped[hlt_path].append(run)
continue
for ls_range in ls_ranges:
l1_prescale = l1_prescale_for_run_ls(l1_seed, run, ls_range)
total[hlt_path][(hlt_prescale, l1_prescale)][run].append(ls_range)
for name, duh in total.iteritems():
for prescale, duh2 in duh.iteritems():
duh[prescale] = dict(duh2)
total[name] = dict(duh)
total = dict(total)
if skipped:
print 'warning: skipped these:'
pprint(skipped)
return total
total_prescales = get_total_prescales(hlt_prescales)
total_prescales_by_intlumi = prescales_by_intlumi(total_prescales)
total_prescales_collapsed = collapse_versions(total_prescales)
total_prescales_collapsed_by_intlumi = prescales_by_intlumi(total_prescales_collapsed)
for x in 'total_prescales total_prescales_by_intlumi total_prescales_collapsed total_prescales_collapsed_by_intlumi'.split():
to_pickle(eval(x), 'prescales_temp/' + x + '.gzpickle')
def analyze(plots=False):
if plots:
set_style()
ps = plot_saver(plot_dir('scaleweights', temp=True), size=(600, 600))
fields = ('nominal', 'n16', 'n84', 'ctl', 'ctlrel', 'ctlrelmd')
fmt = '%-45s' + ' %10s' * len(fields)
print fmt % (('sname', ) + fields)
ss = samples()
for s in ss:
f = ROOT.TFile.Open(s.fn)
s.norm = norm_from_file(f)
s.w = ac.int_lumi_2018 * 1e-3 / s.norm
t = f.Get('mfvMiniTree/t')
hr = draw_hist_register(t, True, True)
myrange = (14, 46) # decorrelated variations - recommended!
#myrange = (6,10) # simple 2x up/down
if plots:
for i in xrange(*myrange):
h = hr.draw('weights[%i]' % i, 'nvtx>=2', binning='100,0,2')
ps.save(str(i))
cut = '(nvtx>=2)' # '(nvtx>=2 && (gen_x[0]**2 + gen_y[0]**2)**0.5<0.08 && (gen_x[1]**2+gen_y[1]**2)**0.5 < 0.08)'
h = hr.draw('weight', cut, binning='1,0,1')
s.nominal, _ = tuple(s.w * x for x in get_integral(h))
s.scaleunc, s.family = [], []
for i in xrange(
*myrange
): # JMTBAD indices into misc_weights are fragile, need to put the pdf weights/info somewhere else than weights
h = hr.draw('weight',
'%s*misc_weights[%i]' % (cut, i),
binning='1,0,1')
v = tuple(s.w * x for x in get_integral(h))[0]
s.family.append(v)
if myrange == (14, 46):
s.family.sort()
s.n16, s.n84 = s.family[5], s.family[27] # since total is 32
s.ctl = (s.n84 - s.n16) / 2
s.avg = avg(s.family)
s.rms = rms(s.family)
s.mdn = (s.family[15] + s.family[16]) / 2
s.md = (s.n84 + s.n16) / 2
#s.relavg = s.scaleunc / s.avg
if s.nominal == 0:
print "s.nominal is 0!!!!!!!!!!!!"
s.nominal = 0.00001
if s.md == 0:
print "s.md is 0!!!!!!!!!!!!"
s.md = 0.00001
s.ctlrel = s.ctl / s.nominal
s.ctlrelmd = s.ctl / s.md
print fmt.replace('%10s', '%10.4f') % (
(s.name, ) + tuple(getattr(s, fi) for fi in fields))
elif myrange == (6, 10):
s.family.sort()
s.n16, s.n84 = s.family[1], s.family[2] # since total is 4
s.ctl = (s.n84 - s.n16) / 2
s.avg = avg(s.family)
s.rms = rms(s.family)
s.mdn = (s.family[1] + s.family[2]) / 2
s.md = (s.n84 + s.n16) / 2
s.ctlrel = s.ctl / s.nominal
s.ctlrelmd = s.ctl / s.md
print fmt.replace('%10s', '%10.4f') % (
(s.name, ) + tuple(getattr(s, fi) for fi in fields))
print "avg %10.4f, rms %10.4f, rel %10.4f" % (s.avg, s.rms,
s.rms / s.avg)
else:
print "invalid range!!!!!!!!!!!!!!!!!"
to_pickle(ss, 'alphaS_parton_shower_uncerts.gzpickle')