def unfold_real_data(self):
logging.info('Unfolding real data')
regularisation = int(self.params['regularisation'].m)
unfold_bg = self.params['unfold_bg'].value
unfold_eff = self.params['unfold_eff'].value
raw_data_0 = self._data['nuall']
if regularisation == 0:
raise AssertionError('Regularisation is set to 0')
# Get the inversed efficiency histogram
if not unfold_eff:
inv_eff = self.get_inv_eff()
# Load response object from disk cache
response = self.create_response()
# Background Subtraction
if unfold_bg:
raw_data_1 = raw_data_0
else:
bg_hist = self.get_bg_hist()
raw_data_1 = raw_data_0 - bg_hist
r_flat = roounfold._flatten_to_1d(raw_data_1)
r_th1d = convert_to_th1d(r_flat, errors=True)
# Unfold
unfold = RooUnfoldBayes(response, r_th1d, regularisation)
unfold.SetVerbose(0)
unfolded_flat = unfold.Hreco(1)
unfold_map = unflatten_thist(in_th1d=unfolded_flat,
binning=self.true_binning,
name=self.output_str,
errors=True)
# Efficiency correction
if not unfold_eff:
unfold_map *= inv_eff
del r_th1d
del unfold
logging.info('Unfolded reco sum {0}'.format(
np.sum(unp.nominal_values(unfold_map.hist))))
return unfold_map
def MyRooUnfold(matrix_name=args.h_matrix, h_reco_getG0_name=args.h_data, h_ptcl_getG0_name = args.h_particle,h_reco_get_bkg_name = args.h_background,outputname=args.h_data+"_unfolded",nrebin = args.nrebin):
rfile_data = TFile(args.rfile_data, 'read')
rfile_particle = TFile(args.rfile_particle, 'read')
rfile_matrix = TFile(args.rfile_matrix, 'read')
rfile_background = TFile(args.rfile_background, 'read')
myfbu = fbu.PyFBU()
myfbu.verbose = True
#GET DATA
h_reco_get = rfile_data.Get(h_reco_getG0_name)
h_reco_get.Rebin(nrebin)
#GET PARTICLE
h_ptcl_get = rfile_particle.Get(h_ptcl_getG0_name)
h_ptcl_get.Rebin(nrebin)
#GET MATRIX
h_response_unf = rfile_matrix.Get(matrix_name)
h_response_unf.ClearUnderflowAndOverflow()
h_response_unf.GetXaxis().SetRange(1, h_response_unf.GetXaxis().GetNbins() )
h_response_unf.GetYaxis().SetRange(1, h_response_unf.GetYaxis().GetNbins() )
h_response_unf.Rebin2D(nrebin,nrebin)
h_response_unf.SetName("Migration_Matrix_simulation")
########### ACCEPTANCY
h_acc = h_response_unf.ProjectionX("reco_recoandparticleX") # Reco M
h_acc.Divide(h_reco_get)
########### AKCEPTANCE saved in h_acc #############
########### EFFICIENCY
h_eff = h_response_unf.ProjectionY("reco_recoandparticleY") # Ptcl M
h_eff.Divide(h_ptcl_get)
h_reco_get_input = rfile_data.Get(h_reco_getG0_name)
h_reco_get_bkg = rfile_background.Get(h_reco_get_bkg_name)
h_reco_get_bkg.Rebin(nrebin)
h_reco_get_input_clone=h_reco_get_input.Clone("")
h_reco_get_input_clone.Add(h_reco_get_bkg,-1)
h_reco_get_input_clone.Multiply(h_acc)
h_reco_or = rfile_data.Get(h_reco_getG0_name)
h_ptcl_or = rfile_particle.Get(h_ptcl_getG0_name)
h_ptcl_or.SetMaximum(h_ptcl_or.GetMaximum()*1.5)
### ROOUNFOLD METHOD ###
m_RooUnfold = RooUnfoldBayes()
m_RooUnfold.SetRegParm( 4 )
m_RooUnfold.SetNToys( 10000 )
m_RooUnfold.SetVerbose( 0 )
m_RooUnfold.SetSmoothing( 0 )
response = RooUnfoldResponse(None, None, h_response_unf, "response", "methods")
m_RooUnfold.SetResponse( response )
m_RooUnfold.SetMeasured( h_reco_get_input_clone )
### SVD METHOD ###
m_RooUnfold_svd = RooUnfoldSvd (response, h_reco_get_input_clone, int(round(h_reco_get_input_clone.GetNbinsX()/2.0,0))) #8
svd_par = int(round(h_reco_get_input_clone.GetNbinsX()/2.0,0))
m_RooUnfold_T = RooUnfoldTUnfold (response, h_reco_get_input_clone) # OR
m_RooUnfold_Ids= RooUnfoldIds (response, h_reco_get_input_clone,int(round(h_reco_get_input_clone.GetNbinsX()/12.0,0))) ## TO DO, SET PARAMETERS TO THE BINNING
Ids_par = int(round(h_reco_get_input_clone.GetNbinsX()/12.0,0))
### FBU METHOD ###
h_response_unf_fbu = TransposeMatrix(h_response_unf)
h_response_unf_fbu_norm = NormalizeResponse(h_response_unf_fbu)
h_response_unf_fbu_norm.SetName("Migration_Matrix_simulation_transpose")
histograms.append(h_response_unf_fbu_norm)
myfbu.response = MakeListResponse(h_response_unf_fbu_norm)
myfbu.data = MakeListFromHisto(h_reco_get_input_clone)
myfbu.lower = []
myfbu.upper = []
h_det_div_ptcl=h_reco_get_input_clone.Clone("")
h_det_div_ptcl.Divide(h_ptcl_or)
h_det_div_ptcl.Divide(h_eff)
h_det_div_ptcl.SetName("det_div_ptcl")
histograms.append(h_det_div_ptcl)
for l in range(len(myfbu.data)):
if ( args.SplitFromBinLow != 0) and ( l+1 <= args.SplitFromBinLow ):
myfbu.lower.append(h_reco_get_input_clone.GetBinContent(l+1)*(2-args.ParameterSplitFromBinLow)*h_det_div_ptcl.GetBinContent(l+1))
myfbu.upper.append(h_reco_get_input_clone.GetBinContent(l+1)*args.ParameterSplitFromBinLow*h_det_div_ptcl.GetBinContent(l+1))
elif ( args.SplitFromBinHigh != 0 ) and ( l+1 >= args.SplitFromBinHigh ):
myfbu.lower.append(h_reco_get_input_clone.GetBinContent(l+1)*(2-args.ParameterSplitFromBinHigh)*h_det_div_ptcl.GetBinContent(l+1))
myfbu.upper.append(h_reco_get_input_clone.GetBinContent(l+1)*args.ParameterSplitFromBinHigh*h_det_div_ptcl.GetBinContent(l+1))
else:
myfbu.lower.append(h_reco_get_input_clone.GetBinContent(l+1)*(2-args.par)*h_det_div_ptcl.GetBinContent(l+1))
myfbu.upper.append(h_reco_get_input_clone.GetBinContent(l+1)*args.par*h_det_div_ptcl.GetBinContent(l+1))
#myfbu.regularization = Regularization('Tikhonov',parameters=[{'refcurv':0.1,'alpha':0.2}]) works for old FBU package and python2.7 and old pymc
myfbu.run()
trace = myfbu.trace
traceName = 'Posterior_1_iteration'
posteriors_diag = MakeTH1Ds(trace, traceName)
h_reco_unfolded, h_reco_unfolded_Mean = MakeUnfoldedHisto(h_reco_or, posteriors_diag)
PlotPosteriors(posteriors_diag,outputname+'_iteration_1')
# EFFICIENCY AND ACCEPTANCY CORRECTIONS
h_reco_unfolded.Divide(h_eff)
h_reco_unfolded_Mean.Divide(h_eff)
h_reco_unfolded_roof = m_RooUnfold.Hreco()
h_reco_unfolded_roof.Divide(h_eff)
h_reco_unfolded_svd = m_RooUnfold_svd.Hreco()
h_reco_unfolded_svd.Divide(h_eff)
h_reco_unfolded_T = m_RooUnfold_T.Hreco()
h_reco_unfolded_T.Divide(h_eff)
h_reco_unfolded_Ids = m_RooUnfold_Ids.Hreco()
h_reco_unfolded_Ids.Divide(h_eff)
PlotRatio(h_reco_unfolded_Mean, h_ptcl_or, h_reco_unfolded_roof, h_reco_unfolded_svd, h_reco_unfolded_T,h_reco_unfolded_Ids, svd_par, Ids_par, outputname+'_iteration_1')
Repeat = True
j = 2
while Repeat:
print("Runnig iteration number: ",j)
myfbu.lower = []
myfbu.upper = []
for l in range(len(myfbu.data)):
posteriors_diag[l].Fit("gaus")
fit = posteriors_diag[l].GetFunction("gaus")
p1 = fit.GetParameter(1)
p2 = fit.GetParameter(2)
myfbu.lower.append(p1-4*p2)
myfbu.upper.append(p1+4*p2)
myfbu.run()
trace = myfbu.trace
traceName = 'Posterior_'+str(j)+'_iteration'
posteriors_diag = MakeTH1Ds(trace, traceName)
h_reco_unfolded, h_reco_unfolded_Mean = MakeUnfoldedHisto(h_reco_or, posteriors_diag)
Repeat = PlotPosteriors(posteriors_diag,outputname+'_iteration_'+str(j))
# EFFICIENCY AND ACCEPTANCY CORRECTIONS
h_reco_unfolded.Divide(h_eff)
h_reco_unfolded_Mean.Divide(h_eff)
h_reco_unfolded_roof = m_RooUnfold.Hreco()
h_reco_unfolded_roof.Divide(h_eff)
h_reco_unfolded_svd = m_RooUnfold_svd.Hreco()
h_reco_unfolded_svd.Divide(h_eff)
h_reco_unfolded_T = m_RooUnfold_T.Hreco()
h_reco_unfolded_T.Divide(h_eff)
h_reco_unfolded_Ids = m_RooUnfold_Ids.Hreco()
h_reco_unfolded_Ids.Divide(h_eff)
PlotRatio(h_reco_unfolded_Mean, h_ptcl_or, h_reco_unfolded_roof, h_reco_unfolded_svd, h_reco_unfolded_T,h_reco_unfolded_Ids, svd_par, Ids_par, outputname+'_iteration_'+str(j))
if j == args.maxiterations:
break
j = j+1
h_reco_unfolded.SetName("result_fbu_fit")
histograms.append(h_reco_unfolded)
h_reco_unfolded_Mean.SetName("result_fbu_Mean")
histograms.append(h_reco_unfolded_Mean)
h_reco_unfolded_roof.SetName("result_roof")
histograms.append(h_reco_unfolded_roof)
h_reco_unfolded_svd.SetName("result_svd")
histograms.append(h_reco_unfolded_svd)
h_reco_unfolded_T.SetName("result_T")
histograms.append(h_reco_unfolded_T)
h_reco_unfolded_Ids.SetName("result_Ids")
histograms.append(h_reco_unfolded_Ids)
h_eff.SetName("efficiency")
histograms.append(h_eff)
h_acc.SetName("acceptancy")
histograms.append(h_acc)
h_reco_or.SetName("reco")
histograms.append(h_reco_or)
h_ptcl_or.SetName("ptcl_simulation")
histograms.append(h_ptcl_or)
h_ratio = h_reco_unfolded.Clone("")
h_ratio.Divide(h_ptcl_or)
h_ratio.SetName("ratio_fbu_fit")
histograms.append(h_ratio)
h_ratio = h_reco_unfolded_Mean.Clone("")
h_ratio.Divide(h_ptcl_or)
h_ratio.SetName("ratio_fbu_Mean")
histograms.append(h_ratio)
h_ratio = h_reco_unfolded_roof.Clone("")
h_ratio.Divide(h_ptcl_or)
h_ratio.SetName("ratio_roof")
histograms.append(h_ratio)
h_ratio = h_reco_unfolded_svd.Clone("")
h_ratio.Divide(h_ptcl_or)
h_ratio.SetName("ratio_svd")
histograms.append(h_ratio)
m_RooUnfold_svd.PrintTable (cout, h_ptcl_or)
m_RooUnfold.PrintTable (cout, h_ptcl_or)
# CORRECTIONS TO GET CROSS SECTION
#DivideBinWidth(h_reco_unfolded_Mean)
#DivideBinWidth(h_reco_unfolded_roof)
#DivideBinWidth(h_reco_unfolded_svd)
#DivideBinWidth(h_reco_unfolded_T)
#DivideBinWidth(h_reco_unfolded_Ids)
#DivideBinWidth(h_ptcl_or)
#Lumi = 36.1e3
#for j in range(1,h_reco_unfolded_Mean.GetXaxis().GetNbins()+1):
# h_reco_unfolded_Mean.SetBinContent(j,h_reco_unfolded_Mean.GetBinContent(j)/(Lumi))
# h_reco_unfolded_Mean.SetBinError(j,h_reco_unfolded_Mean.GetBinError(j)/(Lumi))
# h_reco_unfolded_roof.SetBinContent(j,h_reco_unfolded_roof.GetBinContent(j)/(Lumi))
# h_reco_unfolded_roof.SetBinError(j,h_reco_unfolded_roof.GetBinError(j)/(Lumi))
# h_reco_unfolded_svd.SetBinContent(j,h_reco_unfolded_svd.GetBinContent(j)/(Lumi))
# h_reco_unfolded_svd.SetBinError(j,h_reco_unfolded_svd.GetBinError(j)/(Lumi))
# h_reco_unfolded_T.SetBinContent(j,h_reco_unfolded_T.GetBinContent(j)/(Lumi))
# h_reco_unfolded_T.SetBinError(j,h_reco_unfolded_T.GetBinError(j)/(Lumi))
# h_reco_unfolded_Ids.SetBinContent(j,h_reco_unfolded_Ids.GetBinContent(j)/(Lumi))
# h_reco_unfolded_Ids.SetBinError(j,h_reco_unfolded_Ids.GetBinError(j)/(Lumi))
# h_ptcl_or.SetBinContent(j,h_ptcl_or.GetBinContent(j)/(Lumi))
# h_ptcl_or.SetBinError(j,h_ptcl_or.GetBinError(j)/(Lumi))
#h_reco_unfolded_Mean_clone=h_reco_unfolded_Mean.Clone("FBU_cross_section")
#h_reco_unfolded_roof_clone=h_reco_unfolded_roof.Clone("DAgostini_cross_section")
#h_reco_unfolded_svd_clone=h_reco_unfolded_svd.Clone("SVD_cross_section")
#h_reco_unfolded_T_clone=h_reco_unfolded_T.Clone("TUnfold_cross_section")
#h_reco_unfolded_Ids_clone=h_reco_unfolded_Ids.Clone("Ids_cross_section")
#h_ptcl_or_clone=h_ptcl_or.Clone("Truth_cross_section")
#
#print("CONTROL*******************************************************************: ",h_reco_unfolded_Mean_clone.GetXaxis().GetNbins(),h_reco_unfolded_roof_clone.GetXaxis().GetNbins(),h_reco_unfolded_svd_clone.GetXaxis().GetNbins(),h_reco_unfolded_T_clone.GetXaxis().GetNbins(),h_reco_unfolded_Ids_clone.GetXaxis().GetNbins(),h_ptcl_or_clone.GetXaxis().GetNbins())
#histograms.append(h_reco_unfolded_Mean_clone)
#histograms.append(h_reco_unfolded_roof_clone)
#histograms.append(h_reco_unfolded_svd_clone)
#histograms.append(h_reco_unfolded_T_clone)
#histograms.append(h_reco_unfolded_Ids_clone)
#histograms.append(h_ptcl_or_clone)
SaveHistograms(outputname)
def unfold_mc(self):
logging.debug('Unfolding monte carlo sample')
regularisation = int(self.params['regularisation'].m)
unfold_bg = self.params['unfold_bg'].value
unfold_eff = self.params['unfold_eff'].value
unfold_unweighted = self.params['unfold_unweighted'].value
# Split data into signal, bg and all (signal+bg)
signal_data, bg_data, all_data = self.split_data()
# Load generator level data for signal
gen_data = self.load_gen_data()
# Return true map is regularisation is set to 0
if regularisation == 0:
logging.info('Returning baseline MapSet')
true = roounfold._histogram(events=gen_data,
binning=self.true_binning,
weights=gen_data['pisa_weight'],
errors=True,
name=self.output_str)
return MapSet([true])
# Get the inversed efficiency histogram
if not unfold_eff:
inv_eff = self.get_inv_eff(signal_data, gen_data)
# Set the reco and true data based on cfg file settings
reco_norm_data = None
true_norm_data = None
data = signal_data
if unfold_bg:
reco_norm_data = all_data
if unfold_eff:
true_norm_data = gen_data
if reco_norm_data is None:
reco_norm_data = signal_data
if true_norm_data is None:
true_norm_data = signal_data
if unfold_unweighted:
ones = np.ones(reco_norm_data['pisa_weight'].shape)
reco_norm_data['pisa_weight'] = ones
true_norm_data['pisa_weight'] = ones
data['pisa_weight'] = ones
# Create response object
response = self.create_response(reco_norm_data, true_norm_data, data)
# Make pseduodata
all_hist = self._histogram(events=all_data,
binning=self.reco_binning,
weights=all_data['pisa_weight'],
errors=False,
name='all',
tex=r'\rm{all}')
seed = int(self.params['stat_fluctuations'].m)
if seed != 0:
if self.random_state is None or seed != self.seed:
self.seed = seed
self.random_state = get_random_state(seed)
all_hist = all_hist.fluctuate('poisson', self.random_state)
else:
self.seed = None
self.random_state = None
all_hist.set_poisson_errors()
# Background Subtraction
if unfold_bg:
reco = all_hist
else:
bg_hist = self.get_bg_hist(bg_data)
reco = all_hist - bg_hist
reco.name = 'reco_signal'
reco.tex = r'\rm{reco_signal}'
r_flat = roounfold._flatten_to_1d(reco)
r_th1d = convert_to_th1d(r_flat, errors=True)
# Find optimum value for regularisation parameter
if self.params['optimize_reg'].value:
chisq = None
for r_idx in range(regularisation):
unfold = RooUnfoldBayes(response, r_th1d, r_idx + 1)
unfold.SetVerbose(0)
idx_chisq = unfold.Chi2(self.t_th1d, 1)
if chisq is None:
pass
elif idx_chisq > chisq:
regularisation = r_idx
break
chisq = idx_chisq
# Unfold
unfold = RooUnfoldBayes(response, r_th1d, regularisation)
unfold.SetVerbose(0)
unfolded_flat = unfold.Hreco(1)
unfold_map = unflatten_thist(in_th1d=unfolded_flat,
binning=self.true_binning,
name=self.output_str,
errors=True)
# Efficiency correction
if not unfold_eff:
unfold_map *= inv_eff
del r_th1d
del unfold
logging.info('Unfolded reco sum {0}'.format(
np.sum(unp.nominal_values(unfold_map.hist))))
return unfold_map
def unfoldJetPt(meas, response, xbins):
unfold = RooUnfoldBayes(response, meas)
unfold.SetVerbose(0)
hReco = makeHist(unfold.Hreco(), bins=xbins)
return hReco