parser.add_argument('--sample-def', dest='sample_def', type=str,
help='json file containing the samples definition ')
parser.add_argument('--crab', dest='crab', type=int,
default=0, help='Version of crab to use')
parser.add_argument('--local', dest='local', action='store_true',
default=False, help='Submit to local (NOT SUPPORTED YET)')
args = parser.parse_args()
if not (bool(args.crab) or args.local):
raise ValueError('You did not specify how you want to run! (crab2/3 or local)')
if not os.path.isfile(args.sample_def):
raise ValueError('file %s does not exist' % args.sample_def)
all_samples = [Struct(**i) for i in prettyjson.loads(open(args.sample_def).read())]
to_submit = reduce(
lambda x,y: x+y,
[fnselect(all_samples, pattern, key=lambda x: x.name) for pattern in args.samples],
[]
)
#remove duplicate samples selected by multiple patterns
to_submit = set(to_submit)
jobs = [
Job(
'make_pat_and_ntuples.py',
args.jobid,
sample.name,
sample.DBSName,
def add_category(self, name):
'adds a category'
self.categories[name] = Struct()
def section(self, name):
setattr(self, name, Struct())
def run_module(**kwargs):
args = Struct(**kwargs)
if not args.name:
args.name = args.var
results = [prettyjson.loads(open(i).read())[-1] for i in args.results]
#set_trace()
results.sort(key=lambda x: x['median'])
nevts_graph = plotting.Graph(len(results))
upbound_graph = plotting.Graph(len(results))
max_unc = 0.
bound_range = args.vrange #results[-1]['up_edge'] - results[0]['up_edge']
step = results[1]['up_edge'] - results[0]['up_edge']
#bound_range += step
bound_min = results[0]['up_edge'] - step
for idx, info in enumerate(results):
nevts_graph.SetPoint(idx, info['median'],
info["one_sigma"]["relative"])
upbound_graph.SetPoint(idx, info['up_edge'],
info["one_sigma"]["relative"])
if info["one_sigma"]["relative"] > max_unc:
max_unc = info["one_sigma"]["relative"]
canvas = plotting.Canvas()
nevts_graph.Draw('APL')
nevts_graph.GetXaxis().SetTitle('average number of events')
nevts_graph.GetYaxis().SetTitle('relative fit uncertainty')
canvas.SaveAs(os.path.join(args.outdir, 'nevts_%s.png' % args.name))
tf1 = Plotter.parse_formula(
'scale / (x - shift) + offset',
'scale[1,0,10000],shift[%.2f,%.2f,%.2f],offset[0, 0, 1]' %
(bound_min, bound_min - 2 * step, bound_min + step))
# ROOT.TF1('ret', '[0]/(x - [1])', -3, 1000)
tf1.SetRange(0, 1000)
tf1.SetLineColor(ROOT.EColor.kAzure)
tf1.SetLineWidth(3)
result = upbound_graph.Fit(tf1, 'MES') #WL
scale = tf1.GetParameter('scale')
shift = tf1.GetParameter('shift')
offset = tf1.GetParameter('offset')
upbound_graph.Draw('APL')
upbound_graph.GetXaxis().SetTitle('upper bin edge')
upbound_graph.GetYaxis().SetTitle('relative fit uncertainty')
if args.fullrange:
upbound_graph.GetYaxis().SetRangeUser(offset, max_unc * 1.2)
upbound_graph.GetXaxis().SetRangeUser(shift, (shift + bound_range) * 1.2)
delta = lambda x, y: ((x - shift) / bound_range)**2 + ((y - offset) /
(max_unc - offset))
points = ((bound_min + step) + i * (bound_range / 100.)
for i in xrange(100))
math_best_x = min(points, key=lambda x: delta(x, tf1.Eval(x)))
math_best_y = tf1.Eval(math_best_x)
## math_best_x = math.sqrt(scale*bound_range*(max_unc-offset))+shift
## math_best_y = tf1.Eval(math_best_x) #max_unc*math.sqrt(scale)+offset
best = min(results, key=lambda x: abs(x['up_edge'] - math_best_x))
upbound_best = plotting.Graph(1)
upbound_best.SetPoint(0, best['up_edge'], best["one_sigma"]["relative"])
upbound_best.markerstyle = 29
upbound_best.markersize = 3
upbound_best.markercolor = 2
upbound_best.Draw('P same')
print math_best_x, math_best_y
print best['up_edge'], best["one_sigma"]["relative"]
math_best = plotting.Graph(1)
math_best.SetPoint(0, math_best_x, math_best_y)
math_best.markerstyle = 29
math_best.markersize = 3
math_best.markercolor = ROOT.EColor.kAzure
math_best.Draw('P same')
canvas.SaveAs(os.path.join(args.outdir, 'upbound_%s.png' % args.name))
json = {'best': best['up_edge'], 'unc': best["one_sigma"]["relative"]}
with open(os.path.join(args.outdir, '%s.json' % args.name), 'w') as jfile:
jfile.write(prettyjson.dumps(json))
def run_module(**kwargs):
args = Struct(**kwargs)
mkdir(args.out)
canvas = plotting.Canvas()
pars_regex = None
if args.pars_regex:
pars_regex = re.compile(args.pars_regex)
sample_regex = None
if args.sample_regex:
sample_regex = re.compile(args.sample_regex)
pars_out_regex = None
if args.pars_out_regex:
pars_out_regex = re.compile(args.pars_out_regex)
sample_out_regex = None
if args.sample_out_regex:
sample_out_regex = re.compile(args.sample_out_regex)
output_file = io.root_open('%s/output.root' % args.out, 'recreate')
fpars_tdir = output_file.mkdir('floating_pars')
pulls_tdir = output_file.mkdir('postfit_pulls')
failed_fits = set()
fit_statuses = plotting.Hist(10, -1.5, 8.5)
with io.root_open(args.mlfit) as mlfit:
failed_results = []
passes_results = []
pars = {}
yields = {}
first = True
toys = [i.GetName() for i in mlfit.keys() if i.GetName().startswith('toy_')] if not args.oneshot else [None]
log.info('examining %i toys' % len(toys))
prefit_nuis = None
if args.useprefit:
prefit_nuis = ArgSet(mlfit.nuisances_prefit)
nfailed = 0
for toy in toys:
toy_dir = mlfit.Get(toy) if not args.oneshot else mlfit
keys = set([i.GetName() for i in toy_dir.GetListOfKeys()])
if 'norm_fit_s' not in keys or 'fit_s' not in keys:
log.error('Fit %s failed to produce output!' % toy)
failed_fits.add(toy)
continue
norms = ArgSet(
toy_dir.Get(
'norm_fit_s'
)
)
norms = [i for i in norms]
fit_result = toy_dir.Get(
'fit_s'
)
fit_pars = ArgList(fit_result.floatParsFinal())
if first:
first = False
for i in fit_pars:
if pars_regex and not pars_regex.match(i.GetName()): continue
if pars_out_regex and pars_out_regex.match(i.GetName()): continue
pars[i.GetName()] = []
for i in norms:
if sample_regex and not sample_regex.match(i.GetName()): continue
if sample_out_regex and sample_out_regex.match(i.GetName()): continue
yields[i.GetName()] = []
fit_statuses.Fill(fit_result.status())
fit_failed = any(i.getError() == 0 for i in fit_pars) or fit_result.status() != 0
if fit_failed:
log.error('Fit %s failed to converge properly. It has status %i!' % (toy, fit_result.status()))
nfailed+=1
failed_fits.add(toy)
failed_results.append(fit_result)
continue
passes_results.append(fit_result)
for i in norms:
if i.GetName() in yields:
yields[i.GetName()].append(i)
for i in fit_pars:
if i.GetName() in pars:
pars[i.GetName()].append(i)
if nfailed:
log.error('There were %i fit failed!' % nfailed)
with open('%s/info.txt' % args.out, 'w') as info:
info.write('There were %i fit failed!\n' % nfailed)
fit_statuses.Draw()
canvas.SaveAs('%s/fit_status.png' % args.out)
if not args.nopars:
#Plots the post-fit distribution of the POI and nuisances
out = os.path.join(args.out, 'floating_parameters')
mkdir(out)
for i, j in yields.iteritems():
make_hist(i, j, out, prefix='yield_')
for i, j in pars.iteritems():
make_hist(i, j, out, prefix='par_')
if not args.postpulls:
#Plots the post-fit pulls (nuisance(post) - nuisance(pre))/unc(post)
pulls_dir = os.path.join(args.out, 'postfit_pulls')
mkdir(pulls_dir)
ROOT.gStyle.SetOptFit(11111)
singlenames=set()
for name,value in pars.iteritems():
if pars_regex and not pars_regex.match(name): continue
if pars_out_regex and pars_out_regex.match(i): continue
singlenames.add(get_key(name))
pulls_mean_summary={}
pulls_sigma_summary={}
deltas_mean_summary={}
deltas_sigma_summary={}
for name in singlenames:
nbins = 0
for fullname in pars:
if name in fullname:
nbins = nbins + 1
#print name, nbins
try:
hist = plotting.Hist(nbins, 0.5,nbins+0.5, name = "%s_pull_mean_summary" %name)
pulls_mean_summary[name] = hist
hist = plotting.Hist(nbins, 0.5,nbins+0.5, name = "%s_pull_sigma_summary" %name)
pulls_sigma_summary[name] = hist
hist = plotting.Hist(nbins, 0.5,nbins+0.5, name = "%s_delta_mean_summary" %name)
deltas_mean_summary[name] = hist
hist = plotting.Hist(nbins, 0.5,nbins+0.5, name = "%s_delta_sigma_summary" %name)
deltas_sigma_summary[name] = hist
except:
set_trace()
pulls_mean_summary[ 'all'] = plotting.Hist(len(pars), 0.5, len(pars)+0.5, name = "all_pull_mean_summary" )
pulls_sigma_summary[ 'all'] = plotting.Hist(len(pars), 0.5, len(pars)+0.5, name = "all_pull_sigma_summary" )
deltas_mean_summary[ 'all'] = plotting.Hist(len(pars), 0.5, len(pars)+0.5, name = "all_delta_mean_summary" )
deltas_sigma_summary['all'] = plotting.Hist(len(pars), 0.5, len(pars)+0.5, name = "all_delta_sigma_summary")
for i, j in pars.iteritems():
make_post_distributions(i, j, pulls_dir, pulls_mean_summary, pulls_sigma_summary, prefix='pull_',
dist='pull', prefit=prefit_nuis, tdir=pulls_tdir, skipFit=args.skipFit)
make_post_distributions(i, j, pulls_dir, deltas_mean_summary, deltas_sigma_summary, prefix='delta_',
dist='delta', prefit=prefit_nuis, tdir=pulls_tdir, skipFit=args.skipFit)
for name,histo in pulls_mean_summary.iteritems():
canvas = plotting.Canvas()
histo.Draw()
canvas.Update()
line = ROOT.TLine(histo.GetBinLowEdge(1),0,histo.GetBinLowEdge(histo.GetNbinsX()+1),0)
line.SetLineColor(2)
line.Draw("same")
canvas.Update()
canvas.SaveAs('%s/%s.png' % (pulls_dir,histo.GetName()))
canvas.SaveAs('%s/%s.pdf' % (pulls_dir,histo.GetName()))
pulls_tdir.WriteObject(histo, histo.GetName())
for name,histo in pulls_sigma_summary.iteritems():
canvas = plotting.Canvas()
histo.Draw()
canvas.Update()
line = ROOT.TLine(histo.GetBinLowEdge(1),1,histo.GetBinLowEdge(histo.GetNbinsX()+1),1)
line.SetLineColor(2)
line.Draw("same")
canvas.Update()
canvas.SaveAs('%s/%s.png' % (pulls_dir,histo.GetName()))
canvas.SaveAs('%s/%s.pdf' % (pulls_dir,histo.GetName()))
pulls_tdir.WriteObject(histo, histo.GetName())
for name,histo in deltas_mean_summary.iteritems():
canvas = plotting.Canvas()
histo.Draw()
canvas.Update()
line = ROOT.TLine(histo.GetBinLowEdge(1),0,histo.GetBinLowEdge(histo.GetNbinsX()+1),0)
line.SetLineColor(2)
line.Draw("same")
canvas.Update()
canvas.SaveAs('%s/%s.png' % (pulls_dir,histo.GetName()))
canvas.SaveAs('%s/%s.pdf' % (pulls_dir,histo.GetName()))
pulls_tdir.WriteObject(histo, histo.GetName())
for name,histo in deltas_sigma_summary.iteritems():
histo.Draw()
canvas.Update()
#line = ROOT.TLine(histo.GetBinLowEdge(1),1,histo.GetBinLowEdge(histo.GetNbinsX()+1),1)
#line.Draw("same")
canvas.Update()
canvas.SaveAs('%s/%s.png' % (pulls_dir,histo.GetName()))
canvas.SaveAs('%s/%s.pdf' % (pulls_dir,histo.GetName()))
pulls_tdir.WriteObject(histo, histo.GetName())
if not args.noshapes:
#Overlays the prefit values of the different shapes with the envelope of
#what is fitted by the toys
out = os.path.join(args.out, 'shapes')
mkdir(out)
biased_shapes={}
if args.biasFile:
with io.root_open(args.biasFile) as biased:
biased_dir= biased.prefit \
if hasattr(biased, 'prefit') else \
None
ROOT.TH1.AddDirectory(False)
for key in biased_dir.keys():
biased_shapes[key.name] = asrootpy(key.ReadObj().Clone())
with io.root_open(args.harvested) as harvest:
has_prefit = hasattr(harvest, 'prefit')
prefit = harvest.prefit if has_prefit else None
toys = EnvelopeView(
*[harvest.get(i.GetName()).get(args.variable)
for i in harvest.keys()
if i.GetName().startswith('toy_')
and (i.GetName() not in failed_fits) ]
)
#shapes = [i.GetName() for i in prefit.keys()] #FIXME! should not depend on prefit!
first_toy = [i.GetName() for i in harvest.keys() if i.GetName().startswith('toy_')][0]
not_shapes = set('correlation_matrix')
shapes = [i.GetName() for i in harvest.get(first_toy).get(args.variable).keys() if i.GetName() not in not_shapes]
for shape in shapes:
canvas = plotting.Canvas()
canvas.SetCanvasSize( canvas.GetWw(), int(canvas.GetWh()*1.3) )
upper_pad = plotting.Pad(0, 0.33, 1., 1.)
lower_pad = plotting.Pad(0, 0., 1., 0.33)
upper_pad.set_bottom_margin(0.001)
lower_pad.set_top_margin(0.005)
lower_pad.set_bottom_margin(lower_pad.get_bottom_margin()*3)
upper_pad.Draw()
lower_pad.Draw()
upper_pad.cd()
biased_shape = biased_shapes.get(shape, None)
toy_shape = toys.Get(shape)
pre_shape = None
legend = plotting.Legend(
3+int(has_prefit)+int(bool(biased_shape)),
rightmargin=0.07, topmargin=0.05, leftmargin=0.45)
legend.SetBorderSize(0)
if biased_shape:
biased_shape.title = 'true shape'
biased_shape.legendstyle = 'p'
biased_shape.inlegend = True
biased_shape.drawstyle = 'p'
if has_prefit:
pre_shape = prefit.Get(shape)
pre_shape.title = 'input shape'
pre_shape.legendstyle = 'p'
pre_shape.drawstyle = 'p'
if biased_shape:
pre_shape.legendstyle = 'l'
pre_shape.drawstyle = 'hist'
pre_shape.linecolor = 'blue'
pre_shape.fillstyle = 0
toy_shape.Draw()
if has_prefit:
pre_shape.Draw('same')
if biased_shape:
biased_shape.Draw('same')
legend.AddEntry(toy_shape.two_sigma)
legend.AddEntry(toy_shape.one_sigma)
legend.AddEntry(toy_shape.median)
if has_prefit:
legend.AddEntry(pre_shape)
if biased_shape:
legend.AddEntry(biased_shape)
legend.Draw()
#compute pulls
pulls = None
labelSizeFactor2 = (upper_pad.GetHNDC()+lower_pad.GetHNDC()) / lower_pad.GetHNDC()
labelSizeFactor1 = (upper_pad.GetHNDC()+lower_pad.GetHNDC()) / upper_pad.GetHNDC()
label_factor = labelSizeFactor2/labelSizeFactor1
if has_prefit or biased_shape:
lower_pad.cd()
ref_histo = biased_shape if biased_shape else pre_shape
pulls = toy_shape.median.Clone()
pulls.Reset()
for ref, toy, pull in zip(ref_histo, toy_shape, pulls):
if toy.error == (0.0, 0.0): continue
abs_pull = toy.median-ref.value
#pick correct side of the errors
err = toy.error[1] if abs_pull < 0 else toy.error[0]
pull.value = abs_pull/err
pulls.xaxis.title = args.variable
pulls.yaxis.title = 'pulls'
pulls.set_label_size(ROOT.gStyle.GetLabelSize()*label_factor, "XYZ")
pulls.set_title_size(ROOT.gStyle.GetTitleSize()*label_factor, "XYZ")
pulls.yaxis.set_title_offset(pulls.GetYaxis().GetTitleOffset()/label_factor)
pulls.Draw()
canvas.Update()
canvas.SaveAs('%s/%s.png' % (out, shape))
canvas.SaveAs('%s/%s.pdf' % (out, shape))
with open(os.path.join(out, '%s.json' % shape), 'w') as jfile:
jfile.write(toy_shape.json())
output_file.Close()
def run_module(**kwargs):
args = Struct(**kwargs)
redundant_binning = {}
with open(args.binning) as bins:
redundant_binning = prettyjson.loads(bins.read())
#group binning to merge jet categories
grouping = re.compile('^(?P<base_category>[A-Za-z0-9]+)_\d+Jets$')
binning = {}
for var, categories in redundant_binning.iteritems():
if var not in binning: binning[var] = {}
for category, bin_info in categories.iteritems():
m = grouping.match(category)
if not m:
raise ValueError('Category name %s did not match the regex!' % category)
base = m.group('base_category')
if base not in binning[var]:
binning[var][base] = copy.deepcopy(bin_info)
else:
#make sure that all jet categories have the same bin edges
assert(binning[var][base] == bin_info)
for info in binning.itervalues():
edges = set(
i['low_edge'] for i in info.itervalues()
)
edges.update(
set(
i['up_edge'] for i in info.itervalues()
)
)
edges = sorted(list(edges))
info['edges'] = edges
prefit_norms = {}
with io.root_open(args.input_shape) as shapes:
for key in shapes.keys():
obj = key.ReadObj()
if not obj.InheritsFrom('TDirectory'): continue
for hkey in obj.GetListOfKeys():
hist = hkey.ReadObj()
if not hist.InheritsFrom('TH1'): continue
err = ROOT.Double()
integral = hist.IntegralAndError(
1,
hist.GetNbinsX(),
err
)
val_id = uuid.uuid4().hex
val = ROOT.RooRealVar(val_id, val_id, integral)
val.setError(err)
prefit_norms['%s/%s' % (obj.GetName(), hist.GetName())] = val
with io.root_open(args.fitresult) as results:
dirs = ['']
if args.toys:
dirs = [i.GetName() for i in results.GetListOfKeys() if i.GetName().startswith('toy_')]
postfit_table = Table('Bin:%7s', 'Category:%10s', 'Sample:%20s', 'Yield:%5.1f', 'Error:%5.1f')
postfit_norms = [(i.name, i.value, i.error) for i in results.norm_fit_s]
postfit_norms.sort(key=lambda x: x[0])
for name, val, err in postfit_norms:
bincat, sample = tuple(name.split('/'))
bin, category = tuple(bincat.split('_'))
postfit_table.add_line(bin, category, sample, val, err)
postfit_table.add_separator()
with open(args.out.replace('.root','.raw_txt'), 'w') as out:
out.write(postfit_table.__repr__())
with io.root_open(args.out, 'recreate') as output:
is_prefit_done = False
for dirname in dirs:
input_dir = results.Get(dirname) if dirname else results
if not hasattr(input_dir, 'fit_s'):
continue
fit_result = input_dir.fit_s
pars = asrootpy(fit_result.floatParsFinal())
prefit_pars = asrootpy(fit_result.floatParsInit())
tdir = output
if dirname:
tdir = output.mkdir(dirname)
tdir.cd()
hcorr = asrootpy(fit_result.correlationHist())
par_names = set([i.name for i in pars])
yield_par_names = filter(lambda x: '_FullYield_' in x, par_names)
hcorr.Write()
for observable, info in binning.iteritems():
var_dir = tdir.mkdir(observable)
var_dir.cd()
hists = {}
hists_prefit = {}
for rvar_name in yield_par_names:
category, sample = tuple(rvar_name.split('_FullYield_'))
if category not in info: continue
if sample not in hists:
hists[sample] = plotting.Hist(
info['edges'],
name = sample
)
if not is_prefit_done:
hists_prefit[sample] = plotting.Hist(
info['edges'],
name = sample
)
idx = info[category]['idx']+1
hists[sample][idx].value = pars[rvar_name].value
error = pars[rvar_name].error
hists[sample][idx].error = max(abs(i) for i in error) if isinstance(error, tuple) else error #get max of asym error
if not is_prefit_done:
hists_prefit[sample][idx].value = prefit_pars[rvar_name].value
## Pre-fit floating parameters have no uncertainties
## hists_prefit[sample][idx].error = max(prefit_pars[rvar_name].error)
logging.debug(
'Assigning label %s to bin %i for %s/%s' % (rvar_name, idx, category, sample)
)
hists[sample].xaxis.SetBinLabel(idx, rvar_name)
for h in hists.itervalues():
logging.debug( h.Write() )
if not is_prefit_done:
is_prefit_done = True
output.mkdir('prefit').cd()
for h in hists_prefit.itervalues():
logging.debug( h.Write() )
def run_module(**kwargs):
##################
# DEFINITIONS
##################
opts = Struct(**kwargs)
#when running optimization we do not want to
#plot anything!
if opts.optimize_binning and len(opts.optimize_binning):
opts.noplots = True
if opts.binning and len(opts.binning):
opts.noplots = True
discriminant = 'massDiscr'
phase_space = 'fiducialtight'
full_discr_binning = [2] #range(-15, 16)
jet_categories = [
('0Jets', ['0']),
('1Jets', ['1']),
('2Jets', ['2', '3']),
## ('3Jets', ['3']),
]
## jet_categories = [
## ('0Jets', ['0', '1', '2', '3'])
## ]
## mass_binning = [250., 350., 370., 390., 410., 430., 450., 470., 490., 510., 530., 550., 575., 600., 630., 670., 720., 800., 900, 5000.]
## y_binning = [0., 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 3.]
## ttpt_binning = [0., 20., 30., 40., 50., 60., 70., 90., 110., 140., 180., 250., 1000.]
def flat_bin(width, strt, stop):
ret = []
val = strt
while val <= stop:
ret.append(val)
val += width
return ret
vars_to_unfold = [
Struct(
var='thadpt',
binning=Struct(
gen=flat_bin(40, 0,
500), #[0., 45., 105., 165., 225., 285., 800.],
reco=flat_bin(40, 0,
500), #[20.0*i for i in range(11)]+[800.],
),
xtitle='p_{T}(t_{had}) [GeV]'),
Struct(
var='tleppt',
binning=Struct(
gen=flat_bin(40, 0,
500), #[0., 60., 120., 180., 240., 300., 800.],
reco=flat_bin(40, 0,
500), #[20.0*i for i in range(11)]+[800.],
),
xtitle='p_{T}(t_{lep}) [GeV]'),
Struct(
var='tlepy',
#other_name = 'tleppt',
binning=Struct(
gen=flat_bin(0.2, 0, 2.5), #
reco=flat_bin(0.2, 0, 2.5), #
),
xtitle='|y(t_{lep})|'),
Struct(
var='thady',
binning=Struct(
gen=flat_bin(0.2, 0, 2.5), #
reco=flat_bin(0.2, 0, 2.5), #
),
xtitle='|y(t_{had})|'),
Struct(
var='tty',
binning=Struct(
gen=flat_bin(0.2, 0, 2.5), #
reco=flat_bin(0.2, 0, 2.5), #
),
xtitle='|y(tt)|'),
Struct(
var='ttpt',
binning=Struct(
gen=flat_bin(20, 0,
500), #[0., 45., 105., 165., 225., 285., 800.],
reco=flat_bin(20, 0,
500), #[20.0*i for i in range(11)]+[800.],
),
xtitle='p_{T}(tt) [GeV]'),
Struct(var='ttM',
binning=Struct(
gen=flat_bin(100, 250, 1450),
reco=flat_bin(100, 250, 1450),
),
xtitle='M(tt) [GeV]'),
]
dir_postfix = ''
if opts.optimize_binning and len(opts.optimize_binning):
varname, prev, startbin, binning = tuple(
opts.optimize_binning.split(':'))
if not any(i.var == varname for i in vars_to_unfold):
raise ValueError(
'Sample %s is not among the ones I have to unfold!' % sample)
binning = eval(binning)
prev = eval(prev)
startbin = float(startbin)
info = [i for i in vars_to_unfold if i.var == varname][0]
#use ONLY the variable we want
vars_to_unfold = []
for stop in binning:
clone = info.clone()
clone.binning.reco = prev + [startbin, stop]
clone.dir_postfix = '_'.join(
['%.1f' % i for i in clone.binning.reco])
vars_to_unfold.append(clone)
if opts.binning and len(opts.binning):
varname, binning = tuple(opts.binning.split(':'))
if not any(i.var == varname for i in vars_to_unfold):
raise ValueError(
'Sample %s is not among the ones I have to unfold!' % sample)
binning = eval(binning)
info = [i for i in vars_to_unfold if i.var == varname][0]
info.binning.reco = binning
info.dir_postfix = '_'.join(['%.1f' % i for i in info.binning.reco])
#use ONLY the variable we want
vars_to_unfold = [info]
flumi = None if opts.lumi == -1. else opts.lumi
plotter = TTXSecPlotter(lumi=flumi)
if opts.nodata:
del plotter.views['data']
plotter.views['ttJets_rightAssign'] = {
'view':
plotter.create_tt_subsample(['semilep_visible_right'], 'tt, right cmb',
'#5555ab')
}
plotter.views['ttJets_rightThad'] = {
'view':
plotter.create_tt_subsample(['semilep_right_thad'], 'tt, right t_{h}',
'#aaaad5'),
}
plotter.views['ttJets_rightTlep'] = {
'view':
plotter.create_tt_subsample(['semilep_right_tlep'], 'tt, right t_{l}',
'#ab5555')
}
plotter.views['ttJets_wrongAssign'] = {
'view':
plotter.create_tt_subsample(['semilep_wrong'], 'tt, wrong cmb',
'#d5aaaa')
}
plotter.views['ttJets_other'] = {
'view':
plotter.create_tt_subsample(
['other_tt_decay'],
'Other tt decay',
'#668db3',
)
}
plotter.views['ttJets_wrong'] = {
'view':
urviews.MultifileView(
**{
'':
plotter.create_tt_subsample([
'semilep_wrong',
'other_tt_decay',
'semilep_right_tlep',
], 'tt, wrong cmb', '#ab5555'),
'otherTT_ratio_up':
plotter.create_tt_subsample([
'semilep_wrong',
'other_tt_decay',
'semilep_right_tlep',
],
'tt, wrong cmb',
'#ab5555',
relative_scale=[1., 1.5, 1.]),
'otherTT_ratio_down':
plotter.create_tt_subsample([
'semilep_wrong',
'other_tt_decay',
'semilep_right_tlep',
],
'tt, wrong cmb',
'#ab5555',
relative_scale=[1., 0.5, 1.])
})
}
plotter.mc_samples = [
'QCD*',
'[WZ]Jets',
'single*',
'ttJets_rightThad',
'ttJets_rightTlep',
'ttJets_wrongAssign',
#'ttJets_wrong',
'ttJets_other',
'ttJets_rightAssign',
]
plotter.card_names = {
'vjets': ['[WZ]Jets'],
'single_top': ['single*'],
'tt_right': ['ttJets_rightAssign'],
'tt_wrong': ['ttJets_wrong'],
#'tt_other' : ['ttJets_other'],
'only_thad_right': ['ttJets_rightThad'],
'qcd': ['QCD*'],
}
plotter.systematics = {
'lumi': {
'type': 'lnN',
'samples': ['(?!tt_).*'],
'categories': ['.*'],
'value': 1.05,
},
## 'otherTT_ratio' : {
## 'type' : 'shape',
## 'samples' : ['tt_wrong'],
## 'categories' : ['.*'],
## '+' : lambda x: 'otherTT_ratio_up/%s' % x,
## '-' : lambda x: 'otherTT_ratio_down/%s' % x,
## 'value' : 1.00,
## 'shape_only' : True,
## },
## 'JES' : {
## 'samples' : ['*'],
## 'categories' : ['*'],
## 'type' : 'shape',
## '+' : lambda x: x.replace('nosys', 'jes_up'),
## '-' : lambda x: x.replace('nosys', 'jes_down'),
## 'value' : 1.00,
## },
'MCStat': {
'samples': ['only_thad_right'],
'categories': ['.*'],
'type': 'stat',
'multiplier': 4.,
}
}
##################
# PLOTS
##################
if not opts.noplots:
#cut flow
plotter.cut_flow()
plotter.save('cut_flow')
#rate evolution
plotter.initviews()
lumi = plotter.views['data']['intlumi']
MC_sum = sum(
plotter.make_stack(folder='electrons/nosys').Get('byrun').hists)
electrons_expectation = MC_sum[0].value / lumi
print electrons_expectation
plotter.plot('data', 'electrons/nosys/byrun')
ref_function = ROOT.TF1('f', "%s" % electrons_expectation, 0., 12.)
ref_function.SetLineWidth(3)
ref_function.SetLineStyle(2)
ref_function.Draw('same')
plotter.save('run_evolution_electrons', pdf=False)
MC_sum = sum(
plotter.make_stack(folder='muons/nosys').Get('byrun').hists)
muons_expectation = MC_sum[0].value / lumi
plotter.plot('data', 'muons/nosys/byrun')
ref_function = ROOT.TF1('f', "%s" % muons_expectation, 0., 12.)
ref_function.SetLineWidth(3)
ref_function.SetLineStyle(2)
ref_function.Draw('same')
plotter.save('run_evolution_muons', pdf=False)
plotter.merge_leptons(['muons'])
to_plot = [
('weight', 5, 'event weight', {
'postprocess': lambda x: urviews.OverflowView(x)
}),
('nvtx', 2, '# vertices', {}),
('rho', 5, '#rho', {}),
('lep_pt', 10, 'p_{T}(l) [GeV]', {}),
('lepp_eta', 8, '#eta(l+)', {}),
('lepm_eta', 8, '#eta(l-)', {}),
('bjet_pt', 4, 'p_{T}(b) [GeV]', {}),
('wjet_pt', 4, 'p_{T}(wj) [GeV]', {}),
('bjet_eta', 4, '#eta(b)', {}),
('wjet_eta', 4, '#eta(wj)', {}),
('thadmass', 10, 'M(t_{had})', {
'leftside': True
}),
('whadmass', 10, 'M(W_{had})', {}),
('Mt_W', 10, 'M_{T}(#ell, MET)', {}),
## ("ttM" ,20, 'm(t#bar{t})', {}),
## ("tty" , 4, 'y(t#bar{t})', {}),
## ("ttpt" , 5, 'p_{T}(t#bar{t})', {}),
("costhetastar", 4, '', {}),
("njets", 3, '# of jets', {}),
(discriminant, 2, 'discriminant', {}),
]
plotter.plot_mc_shapes('nosys',
discriminant,
rebin=2,
xaxis=discriminant,
leftside=False,
normalize=True,
show_err=True,
xrange=(-8, 1))
plotter.save('%s_full_shape' % (discriminant), pdf=False)
plotter.plot_mc_shapes('nosys',
discriminant,
rebin=2,
xaxis=discriminant,
leftside=False,
normalize=True,
show_err=True,
xrange=(-8, 1),
use_only=set([
'ttJets_rightTlep', 'ttJets_wrongAssign',
'ttJets_other'
]),
ratio_range=1)
plotter.save('%s_bkg_shape' % (discriminant), pdf=False)
for var, rebin, xaxis, kwargs in to_plot:
leftside = kwargs.get('leftside', False)
if 'leftside' in kwargs:
del kwargs['leftside']
plotter.plot_mc_vs_data('nosys',
var,
leftside=leftside,
rebin=rebin,
xaxis=xaxis,
show_ratio=True,
ratio_range=0.5,
**kwargs)
#set_trace()
#plotter.reset(); plotter.plot_mc_vs_data('nosys', var, leftside=False, rebin=rebin, xaxis=xaxis, show_ratio=True,ratio_range=0.5)
#print var, sum(plotter.keep[0].hists).Integral(), plotter.keep[1].Integral()
plotter.add_cms_blurb(13, lumiformat='%0.3f')
plotter.save(var, pdf=False)
for info in vars_to_unfold:
var = info.var
plotter.plot_mc_vs_data('nosys',
'%s' % var,
leftside=False,
rebin=info.binning.reco,
xaxis=info.xtitle,
show_ratio=True,
ratio_range=0.5)
plotter.add_cms_blurb(13, lumiformat='%0.3f')
plotter.save(var, pdf=False)
previous = info.binning.reco[0]
##plotter.set_subdir('%s/slices' % var)
##for idx, vbin in enumerate(info.binning.reco[1:]):
## plotter.plot_mc_vs_data(
## 'nosys', '%s_%s' % (discriminant, var), leftside=False,
## rebin = full_discr_binning[0],
## xaxis=discriminant,
## preprocess=lambda x: urviews.ProjectionView(x, 'X', [previous, vbin])
## )
## plotter.save('%s_slice_%i' % (discriminant, idx), pdf=False)
##
## plotter.plot_mc_shapes(
## 'nosys', '%s_%s' % (discriminant, var), leftside=False,
## rebin = full_discr_binning[0],
## xaxis=discriminant, normalize=True, show_err=True, xrange=(-8,1),
## preprocess=lambda x: urviews.ProjectionView(x, 'X', [previous, vbin]))
## plotter.save('%s_slice_%i_shape' % (discriminant, idx), pdf=False)
##
## plotter.plot_mc_shapes(
## 'nosys', '%s_%s' % (discriminant, var), leftside=False,
## rebin = full_discr_binning[0],
## xaxis=discriminant, normalize=True, show_err=True, xrange=(-8,1),
## preprocess=lambda x: urviews.ProjectionView(x, 'X', [previous, vbin]),
## use_only=set(['ttJets_rightTlep', 'ttJets_wrongAssign', 'ttJets_other']),
## ratio_range=1)
## plotter.save('%s_bkgslice_%i_shape' % (discriminant, idx), pdf=False)
## previous = vbin
##################
# CARDS
##################
if not opts.noshapes:
plotter.merge_leptons()
for info in vars_to_unfold:
var = info.var
rootpy.log["/"].info('Making cards for %s' % var)
plotter.set_subdir(
os.path.join(
var, opts.subdir,
info.dir_postfix if hasattr(info, 'dir_postfix') else ''))
for category_name, njets in jet_categories:
plotter.write_shapes(
'nosys',
var,
discriminant,
njets,
var_binning=info.binning.reco,
disc_binning=lambda x, *args: full_discr_binning[0],
category_template='Bin%i_' + category_name,
special_cases={
'qcd': plotter.make_single_shape,
'vjets': plotter.make_single_shape
})
plotter.add_systematics()
#plotter.card.add_systematic('lumi', 'lnN', '.*', '[^t]+.*', 1.05)
plotter.save_card(var)
if not opts.nomatrices:
print "\n\n MIGRATION MATRICES \n\n"
########################################
# MIGRATION MATRICES
########################################
plotter.set_subdir('')
fname = os.path.join(plotter.outputdir, 'migration_matrices.root')
with io.root_open(fname, 'recreate') as mfile:
response_dir = 'nosys/response'
for info in vars_to_unfold:
var = info.var
dirname = var
if hasattr(info, 'dir_postfix'):
dirname += '_%s' % info.dir_postfix
mfile.mkdir(dirname).cd()
matrix_path = '%s/%s_matrix' % (response_dir, var)
tt_view = views.SubdirectoryView(
plotter.get_view(plotter.ttbar_to_use, 'unweighted_view'),
'semilep_visible_right')
tt_view = views.SumView(
views.SubdirectoryView(tt_view, 'muons'),
views.SubdirectoryView(tt_view, 'electrons'))
matrix_view_unscaled = plotter.rebin_view(
tt_view, [info.binning.gen, info.binning.reco])
mig_matrix_unscaled = matrix_view_unscaled.Get(matrix_path)
mig_matrix_unscaled.SetName('migration_matrix')
mig_matrix_unscaled.Write()
thruth_unscaled = plotter.rebin_view(
tt_view,
info.binning.gen).Get('%s/%s_truth' % (response_dir, var))
thruth_unscaled.name = 'thruth_unscaled'
thruth_unscaled.Write()
reco_unscaled = plotter.rebin_view(
tt_view,
info.binning.reco).Get('%s/%s_reco' % (response_dir, var))
reco_unscaled.name = 'reco_unscaled'
reco_unscaled.Write()
tt_view = plotter.get_view('ttJets_rightAssign')
matrix_view = plotter.rebin_view(
tt_view, [info.binning.gen, info.binning.reco])
mig_matrix = matrix_view.Get(matrix_path)
mig_matrix.SetName('migration_matrix_scaled')
mig_matrix.Write()
reco_distro = mig_matrix.ProjectionY()
reco_distro.SetName('reco_distribution')
reco_distro.Write()
prefit_view = plotter.rebin_view(
plotter.get_view('ttJets_rightAssign'), info.binning.reco)
plotting.views.SumView.debug = True
prefit_plot = prefit_view.Get('nosys/%s' % var)
prefit_plot.name = 'prefit_distribution'
prefit_plot.Write()
thruth_distro = plotter.rebin_view(
tt_view,
info.binning.gen).Get('%s/%s_truth' % (response_dir, var))
thruth_distro.SetName('true_distribution')
thruth_distro.Write()
logging.info('file: %s written' % fname)