def setUp(self):
# load histograms
# @BROKEN: the file is now in the wrong format!!
self.input_file = File("tests/data/unfolding_merged_asymmetric.root")
self.k_value = 3
self.unfold_method = "TUnfold"
self.met_type = "patType1CorrectedPFMet"
self.variables = ["MET", "WPT", "MT", "ST", "HT"]
self.channels = ["electron", "muon", "combined"]
self.dict = {}
for channel in self.channels:
self.dict[channel] = {}
for variable in self.variables:
self.dict[variable] = {}
h_truth, h_measured, h_response, _ = get_unfold_histogram_tuple(
inputfile=self.input_file, variable=variable, channel=channel, met_type=self.met_type
)
unfolding_object = Unfolding(
h_truth, h_measured, h_response, k_value=self.k_value, method=self.unfold_method
)
tau_unfolding_object = Unfolding(h_truth, h_measured, h_response, tau=100, k_value=-1, method="TUnfold")
self.dict[channel][variable] = {
"h_truth": h_truth,
"h_measured": h_measured,
"h_response": h_response,
"unfolding_object": unfolding_object,
"tau_unfolding_object": tau_unfolding_object,
}
def main():
config = XSectionConfig(13)
# method = 'RooUnfoldSvd'
method = 'RooUnfoldBayes'
file_for_data = File(config.unfolding_powheg_herwig, 'read')
file_for_unfolding = File(config.unfolding_madgraphMLM, 'read')
for channel in ['electron', 'muon', 'combined']:
for variable in config.variables:
tau_value = get_tau_value(config, channel, variable)
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile=file_for_unfolding,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=False,
)
h_data_model, h_data, _, _ = get_unfold_histogram_tuple(
inputfile=file_for_data,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=False,
)
unfolding = Unfolding(h_truth,
h_measured,
h_response,
h_fakes,
method=method,
k_value=-1,
tau=tau_value)
unfolded_data = unfolding.unfold(h_data)
plot_bias(h_truth, h_data_model, unfolded_data, variable, channel,
config.centre_of_mass_energy, method)
def create_toy_mc(input_files, sample, output_folder, n_toy, centre_of_mass, config):
from dps.utils.file_utilities import make_folder_if_not_exists
from dps.utils.toy_mc import generate_toy_MC_from_distribution, generate_toy_MC_from_2Ddistribution
from dps.utils.Unfolding import get_unfold_histogram_tuple
make_folder_if_not_exists(output_folder)
output_file_name = get_output_file_name(output_folder, sample, n_toy, centre_of_mass)
variable_bins = bin_edges_vis.copy()
with root_open(output_file_name, 'recreate') as f_out:
input_file_index = 0
for input_file in input_files:
input_file_hists = File(input_file)
for channel in config.analysis_types.keys():
if channel is 'combined':continue
for variable in variable_bins:
output_dir = f_out.mkdir(str(input_file_index) + '/' + channel + '/' + variable, recurse=True)
cd = output_dir.cd
mkdir = output_dir.mkdir
h_truth, h_measured, h_response, _ = get_unfold_histogram_tuple(input_file_hists,
variable,
channel,
centre_of_mass = centre_of_mass,
visiblePS = True,
load_fakes=False)
cd()
mkdir('Original')
cd ('Original')
h_truth.Write('truth')
h_measured.Write('measured')
h_response.Write('response')
for i in range(1, n_toy+1):
toy_id = 'toy_{0}'.format(i)
mkdir(toy_id)
cd(toy_id)
# create histograms
# add tuples (truth, measured, response) of histograms
truth = generate_toy_MC_from_distribution(h_truth)
measured = generate_toy_MC_from_distribution(h_measured)
response = generate_toy_MC_from_2Ddistribution(h_response)
truth.SetName('truth')
measured.SetName('measured')
response.SetName('response')
truth.Write()
measured.Write()
response.Write()
input_file_index += 1
def main():
config = XSectionConfig(13)
# method = 'RooUnfoldSvd'
method = 'RooUnfoldBayes'
file_for_data = File(config.unfolding_powheg_herwig, 'read')
file_for_unfolding = File(config.unfolding_madgraphMLM, 'read')
for channel in ['electron', 'muon', 'combined']:
for variable in config.variables:
tau_value = get_tau_value(config, channel, variable)
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile=file_for_unfolding,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=False,
)
h_data_model, h_data, _, _ = get_unfold_histogram_tuple(
inputfile=file_for_data,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=False,
)
unfolding = Unfolding(
h_truth, h_measured, h_response, h_fakes,
method=method, k_value=-1, tau=tau_value)
unfolded_data = unfolding.unfold(h_data)
plot_bias(h_truth, h_data_model, unfolded_data, variable, channel,
config.centre_of_mass_energy, method)
def __set_unfolding_histograms__( self ):
# at the moment only one file is supported for the unfolding input
files = set(
[self.truth['file'],
self.gen_vs_reco['file'],
self.measured['file']]
)
if len( files ) > 1:
print "Currently not supported to have different files for truth, gen_vs_reco and measured"
sys.exit()
input_file = files.pop()
visiblePS = self.phaseSpace
t, m, r, f = get_unfold_histogram_tuple(
File(input_file),
self.variable,
self.channel,
centre_of_mass = self.centre_of_mass_energy,
ttbar_xsection=self.measurement_config.ttbar_xsection,
luminosity=self.measurement_config.luminosity,
load_fakes = True,
visiblePS = visiblePS
)
self.h_truth = asrootpy ( t )
self.h_response = asrootpy ( r )
self.h_measured = asrootpy ( m )
self.h_fakes = asrootpy ( f )
self.h_refolded = None
data_file = self.data['file']
if data_file.endswith('.root'):
self.h_data = get_histogram_from_file(self.data['histogram'], self.data['file'])
elif data_file.endswith('.json') or data_file.endswith('.txt'):
data_key = self.data['histogram']
# assume configured bin edges
edges = []
edges = reco_bin_edges_vis[self.variable]
json_input = read_tuple_from_file(data_file)
if data_key == "": # JSON file == histogram
self.h_data = value_error_tuplelist_to_hist(json_input, edges)
else:
self.h_data = value_error_tuplelist_to_hist(json_input[data_key], edges)
else:
print 'Unkown file extension', data_file.split('.')[-1]
def __set_unfolding_histograms__( self ):
# at the moment only one file is supported for the unfolding input
files = set( [self.truth['file'],
self.gen_vs_reco['file'],
self.measured['file']]
)
if len( files ) > 1:
print "Currently not supported to have different files for truth, gen_vs_reco and measured"
sys.exit()
input_file = files.pop()
visiblePS = False
if self.phaseSpace == 'VisiblePS':
visiblePS = True
t, m, r, f = get_unfold_histogram_tuple( File(input_file),
self.variable,
self.channel,
centre_of_mass = self.centre_of_mass_energy,
ttbar_xsection=self.measurement_config.ttbar_xsection,
luminosity=self.measurement_config.luminosity,
load_fakes = True,
visiblePS = visiblePS
)
self.h_truth = asrootpy ( t )
self.h_response = asrootpy ( r )
self.h_measured = asrootpy ( m )
self.h_fakes = asrootpy ( f )
data_file = self.data['file']
if data_file.endswith('.root'):
self.h_data = get_histogram_from_file(self.data['histogram'], self.data['file'])
elif data_file.endswith('.json') or data_file.endswith('.txt'):
data_key = self.data['histogram']
# assume configured bin edges
edges = []
edges = reco_bin_edges_vis[self.variable]
json_input = read_tuple_from_file(data_file)
if data_key == "": # JSON file == histogram
self.h_data = value_error_tuplelist_to_hist(json_input, edges)
else:
self.h_data = value_error_tuplelist_to_hist(json_input[data_key], edges)
else:
print 'Unkown file extension', data_file.split('.')[-1]
def setUp(self):
# load histograms
# @BROKEN: the file is now in the wrong format!!
self.input_file = File('tests/data/unfolding_merged_asymmetric.root')
self.k_value = 3
self.unfold_method = 'TUnfold'
self.met_type = 'patType1CorrectedPFMet'
self.variables = ['MET', 'WPT', 'MT', 'ST', 'HT']
self.channels = ['electron', 'muon', 'combined']
self.dict = {}
for channel in self.channels:
self.dict[channel] = {}
for variable in self.variables:
self.dict[variable] = {}
h_truth, h_measured, h_response, _ = get_unfold_histogram_tuple(
inputfile=self.input_file,
variable=variable,
channel=channel,
met_type=self.met_type)
unfolding_object = Unfolding(h_truth,
h_measured,
h_response,
k_value=self.k_value,
method=self.unfold_method)
tau_unfolding_object = Unfolding(h_truth,
h_measured,
h_response,
tau=100,
k_value=-1,
method='TUnfold')
self.dict[channel][variable] = {
'h_truth': h_truth,
'h_measured': h_measured,
'h_response': h_response,
'unfolding_object': unfolding_object,
'tau_unfolding_object': tau_unfolding_object,
}
def main():
config = XSectionConfig(13)
file_for_powhegPythia = File(config.unfolding_central_firstHalf, 'read')
file_for_ptReweight_up = File(config.unfolding_ptreweight_up_firstHalf, 'read')
file_for_ptReweight_down = File(config.unfolding_ptreweight_down_firstHalf, 'read')
file_for_amcatnlo_pythia8 = File(config.unfolding_amcatnlo_pythia8, 'read')
file_for_powhegHerwig = File(config.unfolding_powheg_herwig, 'read')
file_for_etaReweight_up = File(config.unfolding_etareweight_up, 'read')
file_for_etaReweight_down = File(config.unfolding_etareweight_down, 'read')
file_for_data_template = 'data/normalisation/background_subtraction/13TeV/{variable}/VisiblePS/central/normalisation_{channel}.txt'
for channel in config.analysis_types.keys():
if channel is 'combined':continue
for variable in config.variables:
print variable
# for variable in ['HT']:
# Get the central powheg pythia distributions
_, _, response_central, fakes_central = get_unfold_histogram_tuple(
inputfile=file_for_powhegPythia,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=True,
visiblePS=True
)
measured_central = asrootpy(response_central.ProjectionX('px',1))
truth_central = asrootpy(response_central.ProjectionY())
# Get the reweighted powheg pythia distributions
_, _, response_pt_reweighted_up, _ = get_unfold_histogram_tuple(
inputfile=file_for_ptReweight_up,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True
)
measured_pt_reweighted_up = asrootpy(response_pt_reweighted_up.ProjectionX('px',1))
truth_pt_reweighted_up = asrootpy(response_pt_reweighted_up.ProjectionY())
_, _, response_pt_reweighted_down, _ = get_unfold_histogram_tuple(
inputfile=file_for_ptReweight_down,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True
)
measured_pt_reweighted_down = asrootpy(response_pt_reweighted_down.ProjectionX('px',1))
truth_pt_reweighted_down = asrootpy(response_pt_reweighted_down.ProjectionY())
# _, _, response_eta_reweighted_up, _ = get_unfold_histogram_tuple(
# inputfile=file_for_etaReweight_up,
# variable=variable,
# channel=channel,
# centre_of_mass=13,
# load_fakes=False,
# visiblePS=True
# )
# measured_eta_reweighted_up = asrootpy(response_eta_reweighted_up.ProjectionX('px',1))
# truth_eta_reweighted_up = asrootpy(response_eta_reweighted_up.ProjectionY())
# _, _, response_eta_reweighted_down, _ = get_unfold_histogram_tuple(
# inputfile=file_for_etaReweight_down,
# variable=variable,
# channel=channel,
# centre_of_mass=13,
# load_fakes=False,
# visiblePS=True
# )
# measured_eta_reweighted_down = asrootpy(response_eta_reweighted_down.ProjectionX('px',1))
# truth_eta_reweighted_down = asrootpy(response_eta_reweighted_down.ProjectionY())
# Get the distributions for other MC models
_, _, response_amcatnlo_pythia8, _ = get_unfold_histogram_tuple(
inputfile=file_for_amcatnlo_pythia8,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True
)
measured_amcatnlo_pythia8 = asrootpy(response_amcatnlo_pythia8.ProjectionX('px',1))
truth_amcatnlo_pythia8 = asrootpy(response_amcatnlo_pythia8.ProjectionY())
_, _, response_powhegHerwig, _ = get_unfold_histogram_tuple(
inputfile=file_for_powhegHerwig,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True
)
measured_powhegHerwig = asrootpy(response_powhegHerwig.ProjectionX('px',1))
truth_powhegHerwig = asrootpy(response_powhegHerwig.ProjectionY())
# Get the data input (data after background subtraction, and fake removal)
file_for_data = file_for_data_template.format( variable = variable, channel = channel )
data = read_tuple_from_file(file_for_data)['TTJet']
data = value_error_tuplelist_to_hist( data, reco_bin_edges_vis[variable] )
data = removeFakes( measured_central, fakes_central, data )
# Plot all three
hp = Histogram_properties()
hp.name = 'Reweighting_check_{channel}_{variable}_at_{com}TeV'.format(
channel=channel,
variable=variable,
com='13',
)
v_latex = latex_labels.variables_latex[variable]
unit = ''
if variable in ['HT', 'ST', 'MET', 'WPT', 'lepton_pt']:
unit = ' [GeV]'
hp.x_axis_title = v_latex + unit
hp.x_limits = [ reco_bin_edges_vis[variable][0], reco_bin_edges_vis[variable][-1]]
hp.ratio_y_limits = [0.1,1.9]
hp.ratio_y_title = 'Reweighted / Central'
hp.y_axis_title = 'Number of events'
hp.title = 'Reweighting check for {variable}'.format(variable=v_latex)
measured_central.Rebin(2)
measured_pt_reweighted_up.Rebin(2)
measured_pt_reweighted_down.Rebin(2)
# measured_eta_reweighted_up.Rebin(2)
# measured_eta_reweighted_down.Rebin(2)
measured_amcatnlo_pythia8.Rebin(2)
measured_powhegHerwig.Rebin(2)
data.Rebin(2)
measured_central.Scale( 1 / measured_central.Integral() )
measured_pt_reweighted_up.Scale( 1 / measured_pt_reweighted_up.Integral() )
measured_pt_reweighted_down.Scale( 1 / measured_pt_reweighted_down.Integral() )
measured_amcatnlo_pythia8.Scale( 1 / measured_amcatnlo_pythia8.Integral() )
measured_powhegHerwig.Scale( 1 / measured_powhegHerwig.Integral() )
# measured_eta_reweighted_up.Scale( 1 / measured_eta_reweighted_up.Integral() )
# measured_eta_reweighted_down.Scale( 1/ measured_eta_reweighted_down.Integral() )
data.Scale( 1 / data.Integral() )
print list(measured_central.y())
print list(measured_amcatnlo_pythia8.y())
print list(measured_powhegHerwig.y())
print list(data.y())
compare_measurements(
# models = {'Central' : measured_central, 'PtReweighted Up' : measured_pt_reweighted_up, 'PtReweighted Down' : measured_pt_reweighted_down, 'EtaReweighted Up' : measured_eta_reweighted_up, 'EtaReweighted Down' : measured_eta_reweighted_down},
models = OrderedDict([('Central' , measured_central), ('PtReweighted Up' , measured_pt_reweighted_up), ('PtReweighted Down' , measured_pt_reweighted_down), ('amc@nlo' , measured_amcatnlo_pythia8), ('powhegHerwig' , measured_powhegHerwig) ] ),
measurements = {'Data' : data},
show_measurement_errors=True,
histogram_properties=hp,
save_folder='plots/unfolding/reweighting_check',
save_as=['pdf'],
line_styles_for_models = ['solid','solid','solid','dashed','dashed'],
show_ratio_for_pairs = OrderedDict( [
('PtUpVsCentral' , [ measured_pt_reweighted_up, measured_central ] ),
('PtDownVsCentral' , [ measured_pt_reweighted_down, measured_central ] ),
('amcatnloVsCentral' , [ measured_amcatnlo_pythia8, measured_central ] ),
('powhegHerwigVsCentral' , [ measured_powhegHerwig, measured_central ] ),
('DataVsCentral' , [data, measured_central] )
]),
)
def get_unfolded_normalisation( TTJet_normalisation_results, category, channel, tau_value, visiblePS ):
global com, luminosity, ttbar_xsection, method, variable, path_to_DF
global unfolding_files
files_for_systematics = {
'TTJets_massdown' : unfolding_files['file_for_massdown'],
'TTJets_massup' : unfolding_files['file_for_massup'],
'TTJets_factorisationdown' : unfolding_files['file_for_factorisationdown'],
'TTJets_factorisationup' : unfolding_files['file_for_factorisationup'],
'TTJets_renormalisationdown' : unfolding_files['file_for_renormalisationdown'],
'TTJets_renormalisationup' : unfolding_files['file_for_renormalisationup'],
'TTJets_combineddown' : unfolding_files['file_for_combineddown'],
'TTJets_combinedup' : unfolding_files['file_for_combinedup'],
'TTJets_alphaSdown' : unfolding_files['file_for_alphaSdown'],
'TTJets_alphaSup' : unfolding_files['file_for_alphaSup'],
'TTJets_matchingdown' : unfolding_files['file_for_matchingdown'],
'TTJets_matchingup' : unfolding_files['file_for_matchingup'],
'TTJets_isrdown' : unfolding_files['file_for_isrdown'],
'TTJets_isrup' : unfolding_files['file_for_isrup'],
# 'TTJets_fsrdown' : unfolding_files['file_for_fsrdown'],
'TTJets_fsrup' : unfolding_files['file_for_fsrup'],
'TTJets_uedown' : unfolding_files['file_for_uedown'],
'TTJets_ueup' : unfolding_files['file_for_ueup'],
'TTJets_topPt' : unfolding_files['file_for_topPtSystematic'],
'JES_down' : unfolding_files['file_for_jesdown'],
'JES_up' : unfolding_files['file_for_jesup'],
'JER_down' : unfolding_files['file_for_jerdown'],
'JER_up' : unfolding_files['file_for_jerup'],
'BJet_up' : unfolding_files['file_for_bjetup'],
'BJet_down' : unfolding_files['file_for_bjetdown'],
'LightJet_up' : unfolding_files['file_for_lightjetup'],
'LightJet_down' : unfolding_files['file_for_lightjetdown'],
'TTJets_hadronisation' : unfolding_files['file_for_powheg_herwig'],
'ElectronEnUp' : unfolding_files['file_for_ElectronEnUp'],
'ElectronEnDown' : unfolding_files['file_for_ElectronEnDown'],
'MuonEnUp' : unfolding_files['file_for_MuonEnUp'],
'MuonEnDown' : unfolding_files['file_for_MuonEnDown'],
'TauEnUp' : unfolding_files['file_for_TauEnUp'],
'TauEnDown' : unfolding_files['file_for_TauEnDown'],
'UnclusteredEnUp' : unfolding_files['file_for_UnclusteredEnUp'],
'UnclusteredEnDown' : unfolding_files['file_for_UnclusteredEnDown'],
'Muon_up' : unfolding_files['file_for_LeptonUp'],
'Muon_down' : unfolding_files['file_for_LeptonDown'],
'Electron_up' : unfolding_files['file_for_LeptonUp'],
'Electron_down' : unfolding_files['file_for_LeptonDown'],
'PileUp_up' : unfolding_files['file_for_PUUp'],
'PileUp_down' : unfolding_files['file_for_PUDown'],
'Top_Pt_reweight' : unfolding_files['file_for_ptreweight'],
}
h_truth, h_measured, h_response, h_fakes = None, None, None, None
# Uncertainties by changing the response matrix
if category in files_for_systematics :
print 'Doing category',category,'by changing response matrix'
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile = files_for_systematics[category],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
# PDF Uncertainties
elif category in pdf_uncertainties:
print 'Doing category',category,'by changing response matrix'
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile = unfolding_files['files_for_pdfs'][category],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
# Central and systematics where you just change input MC
else:
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_unfolding'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
# Unfold current normalisation measurements
TTJet_normalisation_results, TTJet_normalisation_results_unfolded, TTJet_normalisation_results_withoutFakes, covariance_matrix = unfold_results(
TTJet_normalisation_results,
category,
channel,
tau_value,
h_truth,
h_measured,
h_response,
h_fakes,
method,
visiblePS,
)
# Store TTJet yields after background subtraction, after background subtraction without fakes and after Unfolding
normalisation_unfolded = {
'TTJet_measured' : TTJet_normalisation_results,
'TTJet_measured_withoutFakes' : TTJet_normalisation_results_withoutFakes,
'TTJet_unfolded' : TTJet_normalisation_results_unfolded,
}
# Return truth of different generators for comparison to data in 04
if category == 'central':
h_truth_massdown, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_massdown'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_massup, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_massup'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
# h_truth_fsrdown, _, _, _ = get_unfold_histogram_tuple(
# inputfile = unfolding_files['file_for_fsrdown'],
# variable = variable,
# channel = channel,
# centre_of_mass = com,
# ttbar_xsection = ttbar_xsection,
# luminosity = luminosity,
# load_fakes = True,
# visiblePS = visiblePS,
# )
h_truth_fsrup, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_fsrup'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_isrdown, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_isrdown'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_isrup, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_isrup'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_uedown, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_uedown'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_ueup, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_ueup'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_powhegPythia8, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_powhegPythia8'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
# h_truth_amcatnlo, _, _, _ = get_unfold_histogram_tuple(
# inputfile = unfolding_files['file_for_amcatnlo'],
# variable = variable,
# channel = channel,
# centre_of_mass = com,
# ttbar_xsection = ttbar_xsection,
# luminosity = luminosity,
# load_fakes = True,
# visiblePS = visiblePS,
# )
# h_truth_madgraphMLM, _, _, _ = get_unfold_histogram_tuple(
# inputfile = unfolding_files['file_for_madgraphMLM'],
# variable = variable,
# channel = channel,
# centre_of_mass = com,
# ttbar_xsection = ttbar_xsection,
# luminosity = luminosity,
# load_fakes = True,
# visiblePS = visiblePS,
# )
h_truth_powheg_herwig, _, _, _ = get_unfold_histogram_tuple(
inputfile = unfolding_files['file_for_powheg_herwig'],
variable = variable,
channel = channel,
centre_of_mass = com,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
normalisation_unfolded['powhegPythia8'] = hist_to_value_error_tuplelist( h_truth_powhegPythia8 )
# normalisation_unfolded['amcatnlo'] = hist_to_value_error_tuplelist( h_truth_madgraphMLM )
# normalisation_unfolded['madgraphMLM'] = hist_to_value_error_tuplelist( h_truth_amcatnlo )
normalisation_unfolded['powhegHerwig'] = hist_to_value_error_tuplelist( h_truth_powheg_herwig )
normalisation_unfolded['massdown'] = hist_to_value_error_tuplelist( h_truth_massdown )
normalisation_unfolded['massup'] = hist_to_value_error_tuplelist( h_truth_massup )
normalisation_unfolded['isrdown'] = hist_to_value_error_tuplelist( h_truth_isrdown )
normalisation_unfolded['isrup'] = hist_to_value_error_tuplelist( h_truth_isrup )
# normalisation_unfolded['fsrdown'] = hist_to_value_error_tuplelist( h_truth_fsrdown )
normalisation_unfolded['fsrup'] = hist_to_value_error_tuplelist( h_truth_fsrup )
normalisation_unfolded['uedown'] = hist_to_value_error_tuplelist( h_truth_uedown )
normalisation_unfolded['ueup'] = hist_to_value_error_tuplelist( h_truth_ueup )
# Write all normalisations in unfolded binning scheme to dataframes
file_template = '{path_to_DF}/{category}/unfolded_normalisation_{channel}_{method}.txt'
write_02(normalisation_unfolded, file_template, path_to_DF, category, channel, method)
return normalisation_unfolded, covariance_matrix
def main():
config = XSectionConfig(13)
file_for_powhegPythia = File(config.unfolding_central, "read")
file_for_ptReweight_up = File(config.unfolding_ptreweight_up, "read")
file_for_ptReweight_down = File(config.unfolding_ptreweight_down, "read")
file_for_etaReweight_up = File(config.unfolding_etareweight_up, "read")
file_for_etaReweight_down = File(config.unfolding_etareweight_down, "read")
file_for_data_template = "data/normalisation/background_subtraction/13TeV/{variable}/VisiblePS/central/normalisation_combined_patType1CorrectedPFMet.txt"
for channel in ["combined"]:
for variable in config.variables:
print variable
# for variable in ['HT']:
# Get the central powheg pythia distributions
_, _, response_central, fakes_central = get_unfold_histogram_tuple(
inputfile=file_for_powhegPythia,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=True,
visiblePS=True,
)
measured_central = asrootpy(response_central.ProjectionX("px", 1))
truth_central = asrootpy(response_central.ProjectionY())
# Get the reweighted powheg pythia distributions
_, _, response_pt_reweighted_up, _ = get_unfold_histogram_tuple(
inputfile=file_for_ptReweight_up,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True,
)
measured_pt_reweighted_up = asrootpy(response_pt_reweighted_up.ProjectionX("px", 1))
truth_pt_reweighted_up = asrootpy(response_pt_reweighted_up.ProjectionY())
_, _, response_pt_reweighted_down, _ = get_unfold_histogram_tuple(
inputfile=file_for_ptReweight_down,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True,
)
measured_pt_reweighted_down = asrootpy(response_pt_reweighted_down.ProjectionX("px", 1))
truth_pt_reweighted_down = asrootpy(response_pt_reweighted_down.ProjectionY())
_, _, response_eta_reweighted_up, _ = get_unfold_histogram_tuple(
inputfile=file_for_etaReweight_up,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True,
)
measured_eta_reweighted_up = asrootpy(response_eta_reweighted_up.ProjectionX("px", 1))
truth_eta_reweighted_up = asrootpy(response_eta_reweighted_up.ProjectionY())
_, _, response_eta_reweighted_down, _ = get_unfold_histogram_tuple(
inputfile=file_for_etaReweight_down,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True,
)
measured_eta_reweighted_down = asrootpy(response_eta_reweighted_down.ProjectionX("px", 1))
truth_eta_reweighted_down = asrootpy(response_eta_reweighted_down.ProjectionY())
# Get the data input (data after background subtraction, and fake removal)
file_for_data = file_for_data_template.format(variable=variable)
data = read_data_from_JSON(file_for_data)["TTJet"]
data = value_error_tuplelist_to_hist(data, reco_bin_edges_vis[variable])
data = removeFakes(measured_central, fakes_central, data)
# Plot all three
hp = Histogram_properties()
hp.name = "Reweighting_check_{channel}_{variable}_at_{com}TeV".format(
channel=channel, variable=variable, com="13"
)
v_latex = latex_labels.variables_latex[variable]
unit = ""
if variable in ["HT", "ST", "MET", "WPT", "lepton_pt"]:
unit = " [GeV]"
hp.x_axis_title = v_latex + unit
hp.y_axis_title = "Number of events"
hp.title = "Reweighting check for {variable}".format(variable=v_latex)
measured_central.Rebin(2)
measured_pt_reweighted_up.Rebin(2)
measured_pt_reweighted_down.Rebin(2)
measured_eta_reweighted_up.Rebin(2)
measured_eta_reweighted_down.Rebin(2)
data.Rebin(2)
measured_central.Scale(1 / measured_central.Integral())
measured_pt_reweighted_up.Scale(1 / measured_pt_reweighted_up.Integral())
measured_pt_reweighted_down.Scale(1 / measured_pt_reweighted_down.Integral())
measured_eta_reweighted_up.Scale(1 / measured_eta_reweighted_up.Integral())
measured_eta_reweighted_down.Scale(1 / measured_eta_reweighted_down.Integral())
data.Scale(1 / data.Integral())
compare_measurements(
models={
"Central": measured_central,
"PtReweighted Up": measured_pt_reweighted_up,
"PtReweighted Down": measured_pt_reweighted_down,
"EtaReweighted Up": measured_eta_reweighted_up,
"EtaReweighted Down": measured_eta_reweighted_down,
},
measurements={"Data": data},
show_measurement_errors=True,
histogram_properties=hp,
save_folder="plots/unfolding/reweighting_check",
save_as=["pdf"],
)
def main():
config = XSectionConfig(13)
method = 'TUnfold'
# A few different files for testing different inputs
file_for_unfolding = File(config.unfolding_central, 'read')
madgraph_file = File(config.unfolding_madgraphMLM, 'read')
for channel in ['combined']:
# for variable in config.variables:
for variable in ['HT']:
print variable
# tau_value = get_tau_value(config, channel, variable)
# tau_value = 0.000228338590921
tau_value = 0.0
# h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
# inputfile=file_for_unfolding,
# variable=variable,
# channel=channel,
# met_type=config.met_type,
# centre_of_mass=config.centre_of_mass_energy,
# ttbar_xsection=config.ttbar_xsection,
# luminosity=config.luminosity,
# load_fakes=False,
# visiblePS=True,
# )
# measured = asrootpy(h_response.ProjectionX('px',1))
# print 'Measured from response :',list(measured.y())
# truth = asrootpy(h_response.ProjectionY())
# print 'Truth from response :',list(truth.y())
h_truth_mad, h_measured_mad, h_response_mad, h_fakes_mad = get_unfold_histogram_tuple(
inputfile=madgraph_file,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
measured = asrootpy(h_response_mad.ProjectionX('px',1))
print 'Measured from response :',list(measured.y())
truth = asrootpy(h_response_mad.ProjectionY())
print 'Truth from response :',list(truth.y())
# Unfold
unfolding = Unfolding( measured,
truth, measured, h_response_mad, None,
method=method, k_value=-1, tau=tau_value)
# unfolded_data = unfolding.closureTest()
# print 'Measured :',list( h_measured.y() )
# h_measured, _ = removeFakes( h_measured, None, h_response)
# for binx in range(0,h_truth.GetNbinsX()+2):
# for biny in range(0,h_truth.GetNbinsX()+2):
# print binx, biny,h_response.GetBinContent(binx,biny)
# print bin,h_truth.GetBinContent(bin)
print 'Tau :',tau_value
unfolded_results = unfolding.unfold()
print 'Unfolded :',list( unfolded_results.y() )
print unfolding.unfoldObject.GetTau()
for channel in ['electron', 'muon']:
taus_from_global_correlaton[channel] = {}
taus_from_L_shape[channel] = {}
k_values = {}
k_values_crosscheck = {}
for variable in variables:
if variable is 'MT': continue
print 'Doing variable', variable, 'in', channel, 'channel'
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile=input_file,
variable=variable,
channel=channel,
met_type=met_type,
centre_of_mass=centre_of_mass,
ttbar_xsection=ttbar_xsection,
luminosity=luminosity,
load_fakes=load_fakes)
# print 'h_fakes = ', h_fakes
h_data = None
if test == 'data':
h_data = get_fit_results_histogram(
data_path=path_to_JSON,
centre_of_mass=centre_of_mass,
channel=channel,
variable=variable,
met_type=met_type,
bin_edges=bin_edges_full[variable])
def main():
config = XSectionConfig(13)
method = 'TUnfold'
# A few different files for testing different inputs
file_for_unfolding = File(config.unfolding_central, 'read')
powheg_herwig_file = File(config.unfolding_powheg_herwig, 'read')
for channel in ['combined', 'muon', 'electron']:
# for variable in config.variables:
for variable in config.variables:
# for variable in ['MET']:
print variable
# tau_value = get_tau_value(config, channel, variable)
# tau_value = 0.000228338590921
tau_value = 0.000
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile=file_for_unfolding,
variable=variable,
channel=channel,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
# measured = asrootpy(h_response.ProjectionX('px',1))
# print 'Measured from response :',list(measured.y())
# truth = asrootpy(h_response.ProjectionY())
# print 'Truth from response :',list(truth.y())
h_truth_ph, h_measured_ph, h_response_ph, h_fakes_ph = get_unfold_histogram_tuple(
inputfile=powheg_herwig_file,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
measured = asrootpy(h_response_ph.ProjectionX('px',1))
# print 'Measured from response :',list(measured.y())
measured.SetBinContent(0,0)
truth = asrootpy(h_response_ph.ProjectionY())
# print 'Truth from response :',list(truth.y())
# print 'Truth underflow :',truth.GetBinContent(0),truth.GetBinContent(truth.GetNbinsX()+1)
# Unfold
unfolding = Unfolding( measured,
truth, measured, h_response, None,
method=method, k_value=-1, tau=tau_value)
# unfolded_data = unfolding.closureTest()
# print 'Measured :',list( h_measured.y() )
# h_measured, _ = removeFakes( h_measured, None, h_response)
# for binx in range(0,h_truth.GetNbinsX()+2):
# for biny in range(0,h_truth.GetNbinsX()+2):
# print binx, biny,h_response.GetBinContent(binx,biny)
# print bin,h_truth.GetBinContent(bin)
# print 'Tau :',tau_value
unfolded_results = unfolding.unfold()
# print 'Unfolded :',list( unfolded_results.y() )
# print unfolding.unfoldObject.GetTau()
# print 'Unfolded :',list( unfolded_results.y() )
refolded_results = unfolding.refold()
refolded_results.rebin(2)
measured.rebin(2)
print 'Refolded :',list( refolded_results.y() )
print 'Measured :',list( measured.y() )
# for i in range(1,refolded_results.GetNbinsX()):
# print i,measured.GetBinContent(i),measured.GetBinError(i),abs( measured.GetBinContent(i) - refolded_results.GetBinContent(i) )
pValue = measured.Chi2Test(refolded_results)
print pValue,1-pValue
def main():
config = XSectionConfig(13)
file_for_powhegPythia = File(config.unfolding_central_firstHalf, 'read')
file_for_ptReweight_up = File(config.unfolding_ptreweight_up_firstHalf,
'read')
file_for_ptReweight_down = File(config.unfolding_ptreweight_down_firstHalf,
'read')
file_for_amcatnlo_pythia8 = File(config.unfolding_amcatnlo_pythia8, 'read')
file_for_powhegHerwig = File(config.unfolding_powheg_herwig, 'read')
file_for_etaReweight_up = File(config.unfolding_etareweight_up, 'read')
file_for_etaReweight_down = File(config.unfolding_etareweight_down, 'read')
file_for_data_template = 'data/normalisation/background_subtraction/13TeV/{variable}/VisiblePS/central/normalisation_{channel}.txt'
for channel in config.analysis_types.keys():
if channel is 'combined': continue
for variable in config.variables:
print variable
# for variable in ['HT']:
# Get the central powheg pythia distributions
_, _, response_central, fakes_central = get_unfold_histogram_tuple(
inputfile=file_for_powhegPythia,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=True,
visiblePS=True)
measured_central = asrootpy(response_central.ProjectionX('px', 1))
truth_central = asrootpy(response_central.ProjectionY())
# Get the reweighted powheg pythia distributions
_, _, response_pt_reweighted_up, _ = get_unfold_histogram_tuple(
inputfile=file_for_ptReweight_up,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True)
measured_pt_reweighted_up = asrootpy(
response_pt_reweighted_up.ProjectionX('px', 1))
truth_pt_reweighted_up = asrootpy(
response_pt_reweighted_up.ProjectionY())
_, _, response_pt_reweighted_down, _ = get_unfold_histogram_tuple(
inputfile=file_for_ptReweight_down,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True)
measured_pt_reweighted_down = asrootpy(
response_pt_reweighted_down.ProjectionX('px', 1))
truth_pt_reweighted_down = asrootpy(
response_pt_reweighted_down.ProjectionY())
# _, _, response_eta_reweighted_up, _ = get_unfold_histogram_tuple(
# inputfile=file_for_etaReweight_up,
# variable=variable,
# channel=channel,
# centre_of_mass=13,
# load_fakes=False,
# visiblePS=True
# )
# measured_eta_reweighted_up = asrootpy(response_eta_reweighted_up.ProjectionX('px',1))
# truth_eta_reweighted_up = asrootpy(response_eta_reweighted_up.ProjectionY())
# _, _, response_eta_reweighted_down, _ = get_unfold_histogram_tuple(
# inputfile=file_for_etaReweight_down,
# variable=variable,
# channel=channel,
# centre_of_mass=13,
# load_fakes=False,
# visiblePS=True
# )
# measured_eta_reweighted_down = asrootpy(response_eta_reweighted_down.ProjectionX('px',1))
# truth_eta_reweighted_down = asrootpy(response_eta_reweighted_down.ProjectionY())
# Get the distributions for other MC models
_, _, response_amcatnlo_pythia8, _ = get_unfold_histogram_tuple(
inputfile=file_for_amcatnlo_pythia8,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True)
measured_amcatnlo_pythia8 = asrootpy(
response_amcatnlo_pythia8.ProjectionX('px', 1))
truth_amcatnlo_pythia8 = asrootpy(
response_amcatnlo_pythia8.ProjectionY())
_, _, response_powhegHerwig, _ = get_unfold_histogram_tuple(
inputfile=file_for_powhegHerwig,
variable=variable,
channel=channel,
centre_of_mass=13,
load_fakes=False,
visiblePS=True)
measured_powhegHerwig = asrootpy(
response_powhegHerwig.ProjectionX('px', 1))
truth_powhegHerwig = asrootpy(response_powhegHerwig.ProjectionY())
# Get the data input (data after background subtraction, and fake removal)
file_for_data = file_for_data_template.format(variable=variable,
channel=channel)
data = read_tuple_from_file(file_for_data)['TTJet']
data = value_error_tuplelist_to_hist(data,
reco_bin_edges_vis[variable])
data = removeFakes(measured_central, fakes_central, data)
# Plot all three
hp = Histogram_properties()
hp.name = 'Reweighting_check_{channel}_{variable}_at_{com}TeV'.format(
channel=channel,
variable=variable,
com='13',
)
v_latex = latex_labels.variables_latex[variable]
unit = ''
if variable in ['HT', 'ST', 'MET', 'WPT', 'lepton_pt']:
unit = ' [GeV]'
hp.x_axis_title = v_latex + unit
hp.x_limits = [
reco_bin_edges_vis[variable][0],
reco_bin_edges_vis[variable][-1]
]
hp.ratio_y_limits = [0.1, 1.9]
hp.ratio_y_title = 'Reweighted / Central'
hp.y_axis_title = 'Number of events'
hp.title = 'Reweighting check for {variable}'.format(
variable=v_latex)
measured_central.Rebin(2)
measured_pt_reweighted_up.Rebin(2)
measured_pt_reweighted_down.Rebin(2)
# measured_eta_reweighted_up.Rebin(2)
# measured_eta_reweighted_down.Rebin(2)
measured_amcatnlo_pythia8.Rebin(2)
measured_powhegHerwig.Rebin(2)
data.Rebin(2)
measured_central.Scale(1 / measured_central.Integral())
measured_pt_reweighted_up.Scale(
1 / measured_pt_reweighted_up.Integral())
measured_pt_reweighted_down.Scale(
1 / measured_pt_reweighted_down.Integral())
measured_amcatnlo_pythia8.Scale(
1 / measured_amcatnlo_pythia8.Integral())
measured_powhegHerwig.Scale(1 / measured_powhegHerwig.Integral())
# measured_eta_reweighted_up.Scale( 1 / measured_eta_reweighted_up.Integral() )
# measured_eta_reweighted_down.Scale( 1/ measured_eta_reweighted_down.Integral() )
data.Scale(1 / data.Integral())
print list(measured_central.y())
print list(measured_amcatnlo_pythia8.y())
print list(measured_powhegHerwig.y())
print list(data.y())
compare_measurements(
# models = {'Central' : measured_central, 'PtReweighted Up' : measured_pt_reweighted_up, 'PtReweighted Down' : measured_pt_reweighted_down, 'EtaReweighted Up' : measured_eta_reweighted_up, 'EtaReweighted Down' : measured_eta_reweighted_down},
models=OrderedDict([
('Central', measured_central),
('PtReweighted Up', measured_pt_reweighted_up),
('PtReweighted Down', measured_pt_reweighted_down),
('amc@nlo', measured_amcatnlo_pythia8),
('powhegHerwig', measured_powhegHerwig)
]),
measurements={'Data': data},
show_measurement_errors=True,
histogram_properties=hp,
save_folder='plots/unfolding/reweighting_check',
save_as=['pdf'],
line_styles_for_models=[
'solid', 'solid', 'solid', 'dashed', 'dashed'
],
show_ratio_for_pairs=OrderedDict([
('PtUpVsCentral',
[measured_pt_reweighted_up, measured_central]),
('PtDownVsCentral',
[measured_pt_reweighted_down, measured_central]),
('amcatnloVsCentral',
[measured_amcatnlo_pythia8, measured_central]),
('powhegHerwigVsCentral',
[measured_powhegHerwig, measured_central]),
('DataVsCentral', [data, measured_central])
]),
)
def main():
args = parse_arguments()
channel = args.channel
variable = args.variable
SetPlotStyle()
config = XSectionConfig(13)
method = 'TUnfold'
files_for_response = [
File(config.unfolding_central, 'read')
]
files_for_toys = [
File(config.unfolding_central, 'read')
]
print variable
tau_value = get_tau_value(config, channel, variable)
print tau_value
pullHistogram = None
for file_for_response in files_for_response:
_, _, h_response, _ = get_unfold_histogram_tuple(
inputfile=file_for_response,
variable=variable,
channel=channel,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
if pullHistogram is None:
pullHistogram = Hist2D( h_response.GetNbinsY(), 1, h_response.GetNbinsY()+1, 1000, -10, 10 )
pullHistogram.SetDirectory(0)
for file_for_toys in files_for_toys:
_, _, h_response_for_toys, _ = get_unfold_histogram_tuple(
inputfile=file_for_toys,
variable=variable,
channel=channel,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
for i in range(0,5000):
if i % 100 == 0: print 'Toy number :',i
toy_response = makeToyResponse( h_response_for_toys.Clone() )
toy_measured = asrootpy(toy_response.ProjectionX('px',1))
toy_truth = asrootpy(h_response_for_toys.ProjectionY())
toy_response_unfolding = makeToyResponse( h_response.Clone() )
toy_response_unfolding.Scale( toy_response.integral(overflow=True) / toy_response_unfolding.integral(overflow=True) )
# Unfold toy data with independent toy response
unfolding = Unfolding( toy_measured,
toy_truth, toy_measured, toy_response_unfolding, None,
method='TUnfold', tau=tau_value)
unfolded_results = unfolding.unfold()
cov, cor, mc_cov = unfolding.get_covariance_matrix()
total_statistical_covariance = cov + mc_cov
for i in range(0,total_statistical_covariance.shape[0] ):
unfolded_results.SetBinError(i+1, np.sqrt( total_statistical_covariance[i,i] ) )
for bin in range(1,unfolded_results.GetNbinsX() + 1 ):
diff = unfolded_results.GetBinContent(bin) - toy_truth.GetBinContent(bin)
pull = diff / unfolded_results.GetBinError( bin )
pullHistogram.Fill( bin, pull )
c = Canvas()
pullHistogram.Draw('COLZ')
plots = r.TObjArray()
# for bin in range(1,pullHistogram.GetNbinsX()):
# slice = pullHistogram.ProjectionY('slice',bin,bin)
# slice.Draw('HIST')
# c.Update()
# slice.Fit('gaus')
# raw_input(bin)
pullHistogram.FitSlicesY(0,0,-1,0,'QNR',plots)
means = None
widths = None
for p in plots:
if p.GetName()[-2:] == '_1':
means = p
elif p.GetName()[-2:] == '_2':
widths = p
means.GetYaxis().SetRangeUser(-2,2)
means.SetMarkerColor(2)
means.SetLineColor(2)
means.GetXaxis().SetTitle(latex_labels.variables_NonLatex[variable])
means.Draw()
widths.SetMarkerColor(4)
widths.SetLineColor(4)
widths.GetXaxis().SetTitle(latex_labels.variables_NonLatex[variable])
widths.Draw('SAME')
l = Legend([], leftmargin=0.45, margin=0.3, topmargin=0.7, entryheight=0.7, entrysep = 0.2)
l.AddEntry( means, 'Pull mean', 'P')
l.AddEntry( widths, 'Pull width', 'P')
l.Draw()
c.Update()
truth_response = asrootpy( h_response.ProjectionY() )
truth_toys = asrootpy( h_response_for_toys.ProjectionY() )
diff_truth = truth_response - truth_toys
outputDir = 'plots/unfolding/pulls/new/'
outputName = '{dir}/{variable}_{channel}.pdf'.format( dir = outputDir, variable = variable, channel = channel)
make_folder_if_not_exists(outputDir)
c.SaveAs(outputName)
def main():
config = XSectionConfig(13)
method = 'TUnfold'
file_for_response = File(config.unfolding_central_secondHalf, 'read')
file_for_powhegPythia = File(config.unfolding_central_firstHalf, 'read')
file_for_ptReweight_up = File(config.unfolding_ptreweight_up_firstHalf, 'read')
file_for_ptReweight_down = File(config.unfolding_ptreweight_down_firstHalf, 'read')
file_for_amcatnlo_pythia8 = File(config.unfolding_amcatnlo_pythia8, 'read')
file_for_powhegHerwig = File(config.unfolding_powheg_herwig, 'read')
file_for_etaReweight_up = File(config.unfolding_etareweight_up, 'read')
file_for_etaReweight_down = File(config.unfolding_etareweight_down, 'read')
samples_and_files_to_compare = {
'Central' : file_for_powhegPythia,
'Nominal' : file_for_response,
'PtReweighted Up' : file_for_ptReweight_up,
'PtReweighted Down' : file_for_ptReweight_down,
# 'amcatnlo_pythia8' : file_for_amcatnlo_pythia8,
# 'powhegHerwig' : file_for_powhegHerwig,
# 'EtaReweighted Up' : file_for_etaReweight_up,
# 'EtaReweighted Down' : file_for_etaReweight_down,
}
for channel in config.analysis_types.keys():
if channel is 'combined':continue
print 'Channel :',channel
for variable in config.variables:
# for variable in ['ST']:
print 'Variable :',variable
# Always unfold with the same response matrix and tau value
tau_value = get_tau_value(config, channel, variable)
# tau_value = 0.00000001
_, _, h_response, _ = get_unfold_histogram_tuple(
inputfile=file_for_response,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
integralOfResponse = asrootpy(h_response.ProjectionY()).integral(0,-1)
# Dictionary to hold results
unfolded_and_truth_for_sample = {}
unfolded_and_truth_xsection_for_sample = {}
for sample, input_file_for_unfolding in samples_and_files_to_compare.iteritems():
_, _, h_response_to_unfold, _ = get_unfold_histogram_tuple(
inputfile=input_file_for_unfolding,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
measured = asrootpy(h_response_to_unfold.ProjectionX('px',1))
truth = asrootpy(h_response_to_unfold.ProjectionY())
scale = integralOfResponse / truth.integral(0,-1)
measured.Scale( scale )
truth.Scale( scale )
# Unfold, and set 'data' to 'measured'
unfolding = Unfolding( measured,
truth, measured, h_response, None,
method=method, tau=tau_value)
unfolded_data = unfolding.unfold()
# unfolded_and_truth_for_sample[sample] = {
# 'truth' : truth_xsection,
# 'unfolded' : unfolded_xsection,
# 'bias' : bias
# }
bias = calculate_bias( truth, unfolded_data )
unfolded_and_truth_for_sample[sample] = {
'truth' : truth,
'unfolded' : unfolded_data,
'bias' : bias
}
unfolded_xsection = calculate_xsection( unfolded_data, variable )
truth_xsection = calculate_xsection( truth, variable )
bias_xsection = calculate_bias( truth_xsection, unfolded_xsection )
unfolded_and_truth_xsection_for_sample[sample] = {
'truth' : truth_xsection,
'unfolded' : unfolded_xsection,
'bias' : bias_xsection
}
plot_closure(unfolded_and_truth_for_sample, variable, channel,
config.centre_of_mass_energy, method, 'number_of_unfolded_events')
plot_closure(unfolded_and_truth_xsection_for_sample, variable, channel,
config.centre_of_mass_energy, method, 'normalised_xsection')
plot_bias(unfolded_and_truth_for_sample, variable, channel,
config.centre_of_mass_energy, method, 'number_of_unfolded_events')
plot_bias(unfolded_and_truth_xsection_for_sample, variable, channel,
config.centre_of_mass_energy, method, 'normalised_xsection', plot_systematics=True)
variables = ['MET', 'WPT', 'MT', 'ST', 'HT']
input_file = File( input_filename_central, 'read' )
input_file_bias = File( input_filename_bias, 'read' )
print 'Performing', test, 'unfolding checks at', centre_of_mass, 'TeV'
for channel in ['electron', 'muon']:
for variable in variables:
print 'Doing', variable, 'variable in', channel, 'channel'
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile = input_file,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = load_fakes )
h_data = None
h_expected = None
if test == 'data':
h_data = get_fit_results_histogram( data_path = path_to_JSON,
centre_of_mass = centre_of_mass,
channel = channel,
variable = variable,
met_type = met_type,
bin_edges = bin_edges_vis[variable] )
h_expected = deepcopy( h_data )
def get_unfolded_normalisation( TTJet_fit_results, category, channel, tau_value, visiblePS ):
global centre_of_mass, luminosity, ttbar_xsection, method
global variable, met_type, path_to_JSON, file_for_unfolding, file_for_powheg_pythia, file_for_powheg_herwig, file_for_ptreweight, files_for_pdfs
global file_for_powhegPythia8, file_for_madgraphMLM, file_for_amcatnlo, file_for_amcatnlo_herwig
# global file_for_matchingdown, file_for_matchingup
global file_for_scaledown, file_for_scaleup
global file_for_massdown, file_for_massup
global ttbar_generator_systematics, ttbar_theory_systematics, pdf_uncertainties
global use_ptreweight
files_for_systematics = {
ttbar_theory_systematic_prefix + 'scaledown' : file_for_scaledown,
ttbar_theory_systematic_prefix + 'scaleup' : file_for_scaleup,
ttbar_theory_systematic_prefix + 'massdown' : file_for_massdown,
ttbar_theory_systematic_prefix + 'massup' : file_for_massup,
ttbar_theory_systematic_prefix + 'factorisationdown' : file_for_factorisationdown,
ttbar_theory_systematic_prefix + 'factorisationup' : file_for_factorisationup,
ttbar_theory_systematic_prefix + 'renormalisationdown' : file_for_renormalisationdown,
ttbar_theory_systematic_prefix + 'renormalisationup' : file_for_renormalisationup,
ttbar_theory_systematic_prefix + 'combineddown' : file_for_combineddown,
ttbar_theory_systematic_prefix + 'combinedup' : file_for_combinedup,
ttbar_theory_systematic_prefix + 'alphaSdown' : file_for_alphaSdown,
ttbar_theory_systematic_prefix + 'alphaSup' : file_for_alphaSup,
'JES_down' : file_for_jesdown,
'JES_up' : file_for_jesup,
'JER_down' : file_for_jerdown,
'JER_up' : file_for_jerup,
'BJet_up' : file_for_bjetup,
'BJet_down' : file_for_bjetdown,
'LightJet_up' : file_for_lightjetup,
'LightJet_down' : file_for_lightjetdown,
ttbar_theory_systematic_prefix + 'hadronisation' : file_for_powheg_herwig,
ttbar_theory_systematic_prefix + 'NLOgenerator' : file_for_amcatnlo,
'ElectronEnUp' : file_for_ElectronEnUp,
'ElectronEnDown' : file_for_ElectronEnDown,
'MuonEnUp' : file_for_MuonEnUp,
'MuonEnDown' : file_for_MuonEnDown,
'TauEnUp' : file_for_TauEnUp,
'TauEnDown' : file_for_TauEnDown,
'UnclusteredEnUp' : file_for_UnclusteredEnUp,
'UnclusteredEnDown' : file_for_UnclusteredEnDown,
'Muon_up' : file_for_LeptonUp,
'Muon_down' : file_for_LeptonDown,
'Electron_up' : file_for_LeptonUp,
'Electron_down' : file_for_LeptonDown,
'PileUp_up' : file_for_PUUp,
'PileUp_down' : file_for_PUDown,
}
h_truth, h_measured, h_response, h_fakes = None, None, None, None
# Systematics where you change the response matrix
if category in files_for_systematics :
print 'Doing category',category,'by changing response matrix'
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple( inputfile = files_for_systematics[category],
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
elif category in pdf_uncertainties:
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple( inputfile = files_for_pdfs[category],
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
# Central and systematics where you just change input MC
else:
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple( inputfile = file_for_unfolding,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
# central_results = hist_to_value_error_tuplelist( h_truth )
TTJet_fit_results_unfolded, TTJet_fit_results_withoutFakes = unfold_results( TTJet_fit_results,
category,
channel,
tau_value,
h_truth,
h_measured,
h_response,
h_fakes,
method,
visiblePS,
)
normalisation_unfolded = {
'TTJet_measured' : TTJet_fit_results,
'TTJet_measured_withoutFakes' : TTJet_fit_results_withoutFakes,
'TTJet_unfolded' : TTJet_fit_results_unfolded
}
#
# THESE ARE FOR GETTING THE HISTOGRAMS FOR COMPARING WITH UNFOLDED DATA
#
if category == 'central':
h_truth_scaledown, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_scaledown,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_scaleup, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_scaleup,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_massdown, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_massdown,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_massup, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_massup,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_powhegPythia8, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_powhegPythia8,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_amcatnlo, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_amcatnlo,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_madgraphMLM, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_madgraphMLM,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
h_truth_powheg_herwig, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_powheg_herwig,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity,
load_fakes = True,
visiblePS = visiblePS,
)
# h_truth_amcatnlo_herwig, _, _, _ = get_unfold_histogram_tuple( inputfile = file_for_amcatnlo_herwig,
# variable = variable,
# channel = channel,
# met_type = met_type,
# centre_of_mass = centre_of_mass,
# ttbar_xsection = ttbar_xsection,
# luminosity = luminosity,
# load_fakes = True,
# visiblePS = visiblePS,
# )
# MADGRAPH_ptreweight_results = hist_to_value_error_tuplelist( h_truth_ptreweight )
# POWHEG_PYTHIA_results = hist_to_value_error_tuplelist( h_truth_POWHEG_PYTHIA )
# MCATNLO_results = None
powhegPythia8_results = hist_to_value_error_tuplelist( h_truth_powhegPythia8 )
madgraphMLM_results = hist_to_value_error_tuplelist( h_truth_madgraphMLM )
amcatnloPythia8_results = hist_to_value_error_tuplelist( h_truth_amcatnlo )
powheg_herwig_results = hist_to_value_error_tuplelist( h_truth_powheg_herwig )
# amcatnlo_herwig_results = hist_to_value_error_tuplelist( h_truth_amcatnlo_herwig )
# matchingdown_results = hist_to_value_error_tuplelist( h_truth_matchingdown )
# matchingup_results = hist_to_value_error_tuplelist( h_truth_matchingup )
scaledown_results = hist_to_value_error_tuplelist( h_truth_scaledown )
scaleup_results = hist_to_value_error_tuplelist( h_truth_scaleup )
massdown_results = hist_to_value_error_tuplelist( h_truth_massdown )
massup_results = hist_to_value_error_tuplelist( h_truth_massup )
normalisation_unfolded['powhegPythia8'] = powhegPythia8_results
normalisation_unfolded['amcatnlo'] = amcatnloPythia8_results
normalisation_unfolded['madgraphMLM'] = madgraphMLM_results
normalisation_unfolded['powhegHerwig'] = powheg_herwig_results
# normalisation_unfolded['amcatnloHerwig'] = amcatnlo_herwig_results
normalisation_unfolded['scaledown'] = scaledown_results
normalisation_unfolded['scaleup'] = scaleup_results
normalisation_unfolded['massdown'] = massdown_results
normalisation_unfolded['massup'] = massup_results
return normalisation_unfolded
def main():
config = XSectionConfig(13)
method = 'TUnfold'
file_for_response = File(config.unfolding_central, 'read')
file_for_powhegPythia = File(config.unfolding_central, 'read')
file_for_madgraph = File(config.unfolding_madgraphMLM, 'read')
file_for_amcatnlo = File(config.unfolding_amcatnlo, 'read')
file_for_ptReweight_up = File(config.unfolding_ptreweight_up, 'read')
file_for_ptReweight_down = File(config.unfolding_ptreweight_down, 'read')
file_for_etaReweight_up = File(config.unfolding_etareweight_up, 'read')
file_for_etaReweight_down = File(config.unfolding_etareweight_down, 'read')
samples_and_files_to_compare = {
'Central' : file_for_powhegPythia,
'PtReweighted Up' : file_for_ptReweight_up,
'PtReweighted Down' : file_for_ptReweight_down,
'EtaReweighted Up' : file_for_etaReweight_up,
'EtaReweighted Down' : file_for_etaReweight_down,
'Madgraph' : file_for_madgraph,
'amc@NLO' : file_for_amcatnlo
}
for channel in ['combined']:
for variable in config.variables:
# for variable in ['ST']:
print 'Variable :',variable
# Always unfold with the same response matrix and tau value
tau_value = get_tau_value(config, channel, variable)
_, _, h_response, _ = get_unfold_histogram_tuple(
inputfile=file_for_response,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
integralOfResponse = asrootpy(h_response.ProjectionY()).integral(0,-1)
# Dictionary to hold results
unfolded_and_truth_for_sample = {}
for sample, input_file_for_unfolding in samples_and_files_to_compare.iteritems():
_, _, h_response_to_unfold, _ = get_unfold_histogram_tuple(
inputfile=input_file_for_unfolding,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=True,
)
measured = asrootpy(h_response_to_unfold.ProjectionX('px',1))
truth = asrootpy(h_response_to_unfold.ProjectionY())
scale = integralOfResponse / truth.integral(0,-1)
measured.Scale( scale )
truth.Scale( scale )
# Unfold, and set 'data' to 'measured'
unfolding = Unfolding( measured,
truth, measured, h_response, None,
method=method, k_value=-1, tau=tau_value)
unfolded_data = unfolding.unfold()
unfolded_xsection = calculate_xsection( unfolded_data, variable )
truth_xsection = calculate_xsection( truth, variable )
bias = calculate_bias( truth, unfolded_data )
unfolded_and_truth_for_sample[sample] = {
'truth' : truth_xsection,
'unfolded' : unfolded_xsection,
'bias' : bias
}
plot_closure(unfolded_and_truth_for_sample, variable, channel,
config.centre_of_mass_energy, method)
plot_bias(unfolded_and_truth_for_sample, variable, channel,
config.centre_of_mass_energy, method)
# taus_from_L_shape = {}
for variable in variables:
# taus_from_global_correlaton[channel] = {}
# taus_from_L_shape[channel] = {}
for channel in ['electron', 'muon']:
if channel == 'electron':
used_k = measurement_config.k_values_electron[variable]
if channel == 'muon':
used_k = measurement_config.k_values_muon[variable]
print 'Doing variable"', variable, '" in', channel, 'channel'
h_truth, h_measured, h_response, _ = get_unfold_histogram_tuple(
inputfile = input_file,
variable = variable,
channel = channel,
met_type = met_type,
centre_of_mass = centre_of_mass,
ttbar_xsection = ttbar_xsection,
luminosity = luminosity )
h_data = None
if use_data:
h_data = get_data_histogram( channel, variable, met_type )
else:
h_data = deepcopy( h_measured )
tau, rho, tau_values, rho_values, tau_from_k = get_tau_from_global_correlation( h_truth, h_measured, h_response, h_data )
draw_global_correlation( tau_values, rho_values, tau, rho, channel, variable, tau_from_k )
#tau, l_curve, x, y = get_tau_from_L_shape( h_truth, h_measured, h_response, h_data )
