def calculate_xsections(normalisation, category, channel): global variable, met_type, path_to_JSON # calculate the x-sections branching_ratio = 0.15 if channel == 'combined': branching_ratio = branching_ratio * 2 TTJet_xsection = calculate_xsection(normalisation['TTJet_measured'], luminosity, branching_ratio) # L in pb1 TTJet_xsection_unfolded = calculate_xsection(normalisation['TTJet_unfolded'], luminosity, branching_ratio) # L in pb1 MADGRAPH_xsection = calculate_xsection(normalisation['MADGRAPH'], luminosity, branching_ratio) # L in pb1 POWHEG_xsection = calculate_xsection(normalisation['POWHEG'], luminosity, branching_ratio) # L in pb1 MCATNLO_xsection = calculate_xsection(normalisation['MCATNLO'], luminosity, branching_ratio) # L in pb1 matchingdown_xsection = calculate_xsection(normalisation['matchingdown'], luminosity, branching_ratio) # L in pb1 matchingup_xsection = calculate_xsection(normalisation['matchingup'], luminosity, branching_ratio) # L in pb1 scaledown_xsection = calculate_xsection(normalisation['scaledown'], luminosity, branching_ratio) # L in pb1 scaleup_xsection = calculate_xsection(normalisation['scaleup'], luminosity, branching_ratio) # L in pb1 xsection_unfolded = {'TTJet_measured' : TTJet_xsection, 'TTJet_unfolded' : TTJet_xsection_unfolded, 'MADGRAPH': MADGRAPH_xsection, 'POWHEG': POWHEG_xsection, 'MCATNLO': MCATNLO_xsection, # systematics 'matchingdown': matchingdown_xsection, 'matchingup': matchingup_xsection, 'scaledown': scaledown_xsection, 'scaleup': scaleup_xsection } write_data_to_JSON(xsection_unfolded, path_to_JSON + '/xsection_measurement_results' + '/kv' + str(unfoldCfg.SVD_k_value) + '/' + category + '/xsection_' + channel + '_' + met_type + '.txt')
def write_normalised_xsection_measurement(options, measurement, measurement_unfolded, summary = '' ): ''' Writes the list of normalised measurements and normalised unfolded measurements of the form: [Central Value, Lower Systemtic, Upper Systematic] to a json. Different combinations of systematic uncertainty are stored as different json by appending different 'summary' ''' path_to_JSON=options['path_to_JSON'] method=options['method'] channel=options['channel'] output_file = '{path_to_JSON}/TEST/central/normalised_xsection_{channel}_{method}_with_errors.txt' output_file = output_file.format( path_to_JSON = path_to_JSON, channel = channel, method = method, ) if not summary == '': output_file = output_file.replace( 'with_errors', summary + '_errors' ) output = {'TTJet_measured':measurement, 'TTJet_unfolded': measurement_unfolded} write_data_to_JSON( output, output_file ) print("Data written to : ", output_file)
def calculate_normalised_xsections(normalisation, category, channel, normalise_to_one=False): global variable, met_type, path_to_JSON TTJet_normalised_xsection = calculate_normalised_xsection(normalisation['TTJet_measured'], bin_widths[variable], normalise_to_one) TTJet_normalised_xsection_unfolded = calculate_normalised_xsection(normalisation['TTJet_unfolded'], bin_widths[variable], normalise_to_one) MADGRAPH_normalised_xsection = calculate_normalised_xsection(normalisation['MADGRAPH'], bin_widths[variable], normalise_to_one) POWHEG_normalised_xsection = calculate_normalised_xsection(normalisation['POWHEG'], bin_widths[variable], normalise_to_one) MCATNLO_normalised_xsection = calculate_normalised_xsection(normalisation['MCATNLO'], bin_widths[variable], normalise_to_one) matchingdown_normalised_xsection = calculate_normalised_xsection(normalisation['matchingdown'], bin_widths[variable], normalise_to_one) matchingup_normalised_xsection = calculate_normalised_xsection(normalisation['matchingup'], bin_widths[variable], normalise_to_one) scaledown_normalised_xsection = calculate_normalised_xsection(normalisation['scaledown'], bin_widths[variable], normalise_to_one) scaleup_normalised_xsection = calculate_normalised_xsection(normalisation['scaleup'], bin_widths[variable], normalise_to_one) normalised_xsection = {'TTJet_measured' : TTJet_normalised_xsection, 'TTJet_unfolded' : TTJet_normalised_xsection_unfolded, 'MADGRAPH': MADGRAPH_normalised_xsection, 'POWHEG': POWHEG_normalised_xsection, 'MCATNLO': MCATNLO_normalised_xsection, # systematics 'matchingdown': matchingdown_normalised_xsection, 'matchingup': matchingup_normalised_xsection, 'scaledown': scaledown_normalised_xsection, 'scaleup': scaleup_normalised_xsection } filename = path_to_JSON + '/xsection_measurement_results' + '/kv' + str(unfoldCfg.SVD_k_value) + '/' + category + '/normalised_xsection_' + channel + '_' + met_type + '.txt' if normalise_to_one: filename = filename.replace('normalised_xsection', 'normalised_to_one_xsection') write_data_to_JSON(normalised_xsection, filename)
def write_normalised_xsection_measurement(measurement, measurement_unfolded, channel, summary=''): global path_to_JSON, met_type output_file = path_to_JSON + 'central/normalised_xsection_' + channel + '_' + met_type + '_with_errors.txt' if not summary == '': output_file = output_file.replace('with_errors', summary + '_errors') output = {'TTJet_measured':measurement, 'TTJet_unfolded': measurement_unfolded} write_data_to_JSON(output, output_file)
def save_pulls(pulls, method, channel, tau_value, output_folder): ''' Saves pull distributions in JSON format ''' file_template = 'Pull_data_%s_%s_%.4g.txt' output_file = output_folder + \ file_template % (method, channel, tau_value) write_data_to_JSON(pulls, output_file) print('Pulls saved in file: ', output_file) return output_file
def main(): global config, options parser = OptionParser() parser.add_option("-p", "--path", dest="path", default='data/fit_checks/no_merging', help="set path to JSON files") parser.add_option('--create_fit_data', dest="create_fit_data", action="store_true", help="create the fit data for testing.") parser.add_option('--refit', dest="refit", action="store_true", help="Fit again even if the output already exists") parser.add_option('--test', dest="test", action="store_true", help="Test only: run just one selected sample") variables = config.histogram_path_templates.keys() fit_variables = fit_var_inputs mc_samples = ['TTJet', 'SingleTop', 'QCD', 'V+Jets'] tests = closure_tests channels = ['electron', 'muon'] COMEnergies = ['7', '8'] (options, _) = parser.parse_args() print 'Running from path', options.path if (options.create_fit_data): create_fit_data(options.path, variables, fit_variables, mc_samples, COMEnergies=COMEnergies, channels=channels) output_file = options.path + '/fit_test_output.txt' if options.test: output_file = options.path + '/fit_test_output_test.txt' if options.refit or not os.path.isfile(output_file) or options.test: if os.path.isfile(options.path + '/fit_check_data.txt'): fit_data = read_data_from_JSON(options.path + '/fit_check_data.txt') results = run_tests(fit_data, COMEnergies, variables, fit_variables, mc_samples, channels, tests) write_data_to_JSON(results, output_file) else: print 'Please run bin/prepare_data_for_fit_checks first' print 'Then run this script with the option --create_fit_data.' results = read_data_from_JSON(output_file) plot_results(results)
def get_unfolded_normalisation(TTJet_fit_results, category, channel): global variable, met_type, path_to_JSON, file_for_unfolding, file_for_powheg, file_for_mcatnlo global file_for_matchingdown, file_for_matchingup, file_for_scaledown, file_for_scaleup global ttbar_generator_systematics files_for_systematics = { ttbar_theory_systematic_prefix + 'matchingdown':file_for_matchingdown, ttbar_theory_systematic_prefix + 'matchingup':file_for_matchingup, ttbar_theory_systematic_prefix + 'scaledown':file_for_scaledown, ttbar_theory_systematic_prefix + 'scaleup':file_for_scaleup, } h_truth, h_measured, h_response = None, None, None if category in ttbar_generator_systematics and not 'ptreweight' in category: h_truth, h_measured, h_response = get_unfold_histogram_tuple(files_for_systematics[category], variable, channel, met_type) elif 'mcatnlo_matrix' in category: h_truth, h_measured, h_response = get_unfold_histogram_tuple(file_for_mcatnlo, variable, channel, met_type) else: h_truth, h_measured, h_response = get_unfold_histogram_tuple(file_for_unfolding, variable, channel, met_type) MADGRAPH_results = hist_to_value_error_tuplelist(h_truth) POWHEG_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_powheg, variable, channel, met_type)[0]) MCATNLO_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_mcatnlo, variable, channel, met_type)[0]) matchingdown_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_matchingdown, variable, channel, met_type)[0]) matchingup_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_matchingup, variable, channel, met_type)[0]) scaledown_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_scaledown, variable, channel, met_type)[0]) scaleup_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_scaleup, variable, channel, met_type)[0]) TTJet_fit_results_unfolded = unfold_results(TTJet_fit_results, category, channel, h_truth, h_measured, h_response, 'RooUnfoldSvd' # 'TSVDUnfold' ) normalisation_unfolded = { 'TTJet_measured' : TTJet_fit_results, 'TTJet_unfolded' : TTJet_fit_results_unfolded, 'MADGRAPH': MADGRAPH_results, # other generators 'POWHEG': POWHEG_results, 'MCATNLO': MCATNLO_results, # systematics 'matchingdown': matchingdown_results, 'matchingup': matchingup_results, 'scaledown': scaledown_results, 'scaleup': scaleup_results } write_data_to_JSON(normalisation_unfolded, path_to_JSON + '/xsection_measurement_results' + '/kv' + str(unfoldCfg.SVD_k_value) + '/' + category + '/normalisation_' + channel + '_' + met_type + '.txt') return normalisation_unfolded
def create_fit_data( path, variables, fit_variables, mc_samples, COMEnergies, channels ): ''' Creates the fit data in path + fit_check_data.txt in the JSON format. The resulting dictionary is of the form {centre-of-mass-energy: { channel : { variable : { variable_bin : { fit_variable : { templates : { sample : []}, initial_values : { sample : []} } } } } } } ''' # first step is to read the data raw_data = {} # the folder structure is # path/fit_variable/centre-of-mass-energy/variable/fit_results/central/* for fit_variable in fit_variables: raw_data[fit_variable] = {} for COMEnergy in COMEnergies: raw_data[fit_variable][COMEnergy] = {} for variable in variables: raw_data[fit_variable][COMEnergy][variable] = {} for channel in channels: raw_data[fit_variable][COMEnergy][variable][channel] = {} data_path = path + '/' + fit_variable + '/' + str( COMEnergy ) + 'TeV' templates = read_fit_templates( data_path, variable, channel = channel ) initial_values = read_fit_input( data_path, variable, channel = channel ) raw_data[fit_variable][COMEnergy][variable][channel]['templates'] = templates raw_data[fit_variable][COMEnergy][variable][channel]['initial_values'] = initial_values # put it into the new structure fit_data = {} for COMEnergy in COMEnergies: fit_data[COMEnergy] = {} for channel in channels: fit_data[COMEnergy][channel] = {} for variable in variables: fit_data[COMEnergy][channel][variable] = {} for v_bin in variable_bins_ROOT[variable]: fit_data[COMEnergy][channel][variable][v_bin] = {} for fit_variable in fit_variables: fit_data[COMEnergy][channel][variable][v_bin][fit_variable] = {} fit_data[COMEnergy][channel][variable][v_bin][fit_variable]['templates'] = raw_data[fit_variable][COMEnergy][variable][channel]['templates'] fit_data[COMEnergy][channel][variable][v_bin][fit_variable]['initial_values'] = raw_data[fit_variable][COMEnergy][variable][channel]['initial_values'] write_data_to_JSON( fit_data, path + '/fit_check_data.txt', indent = False )
def save_pulls(pulls, test, method, channel, output_folder, n_toy_mc, n_toy_data, offset_toy_mc, offset_toy_data): ''' Saves pull distributions in JSON format ''' file_template = 'Pulls_%s_%s_%s_toy_MC_%d_to_%d_MC_%d_to_%d_data.txt' output_file = output_folder + \ file_template % (test, method, channel, offset_toy_mc + 1, n_toy_mc + offset_toy_mc, offset_toy_data + 1, n_toy_data + offset_toy_data) write_data_to_JSON(pulls, output_file) print('Pulls saved in file: ', output_file)
def write_normalised_xsection_measurement( measurement, measurement_unfolded, channel, summary = '' ): global path_to_JSON, met_type, k_values output_file = path_to_JSON + '/' + channel + '/kv' + str( k_values[channel] ) + '/central/normalised_xsection_' + met_type + '_with_errors.txt' if channel == 'combined': output_file = output_file.replace( 'kv' + str( k_values[channel] ), '' ) if not summary == '': output_file = output_file.replace( 'with_errors', summary + '_errors' ) output = {'TTJet_measured':measurement, 'TTJet_unfolded': measurement_unfolded} write_data_to_JSON( output, output_file )
def calculate_normalised_xsections( normalisation, category, channel, normalise_to_one = False ): global variable, met_type, path_to_JSON, phase_space binWidths = None if phase_space == 'VisiblePS': binWidths = bin_widths_visiblePS elif phase_space == 'FullPS': binWidths = bin_widths TTJet_normalised_xsection = calculate_normalised_xsection( normalisation['TTJet_measured'], binWidths[variable], normalise_to_one ) TTJet_withoutFakes_normalised_xsection = calculate_normalised_xsection( normalisation['TTJet_measured_withoutFakes'], binWidths[variable], normalise_to_one ) TTJet_normalised_xsection_unfolded = calculate_normalised_xsection( normalisation['TTJet_unfolded'], binWidths[variable], normalise_to_one ) normalised_xsection = {'TTJet_measured' : TTJet_normalised_xsection, 'TTJet_measured_withoutFakes' : TTJet_withoutFakes_normalised_xsection, 'TTJet_unfolded' : TTJet_normalised_xsection_unfolded } if category == 'central': powhegPythia8_normalised_xsection = calculate_normalised_xsection( normalisation['powhegPythia8'], binWidths[variable], normalise_to_one ) amcatnlo_normalised_xsection = calculate_normalised_xsection( normalisation['amcatnlo'], binWidths[variable], normalise_to_one ) powhegHerwig_normalised_xsection = calculate_normalised_xsection( normalisation['powhegHerwig'], binWidths[variable], normalise_to_one ) amcatnloHerwig_normalised_xsection = calculate_normalised_xsection( normalisation['amcatnloHerwig'], binWidths[variable], normalise_to_one ) madgraphMLM_normalised_xsection = calculate_normalised_xsection( normalisation['madgraphMLM'], binWidths[variable], normalise_to_one ) scaledown_normalised_xsection = calculate_normalised_xsection( normalisation['scaledown'], binWidths[variable], normalise_to_one ) scaleup_normalised_xsection = calculate_normalised_xsection( normalisation['scaleup'], binWidths[variable], normalise_to_one ) massdown_normalised_xsection = calculate_normalised_xsection( normalisation['massdown'], binWidths[variable], normalise_to_one ) massup_normalised_xsection = calculate_normalised_xsection( normalisation['massup'], binWidths[variable], normalise_to_one ) normalised_xsection['powhegPythia8'] = powhegPythia8_normalised_xsection normalised_xsection['amcatnlo'] = amcatnlo_normalised_xsection normalised_xsection['madgraphMLM' ] = madgraphMLM_normalised_xsection normalised_xsection['powhegHerwig'] = powhegHerwig_normalised_xsection normalised_xsection['amcatnloHerwig'] = amcatnloHerwig_normalised_xsection normalised_xsection['scaledown'] = scaledown_normalised_xsection normalised_xsection['scaleup'] = scaleup_normalised_xsection normalised_xsection['massdown'] = massdown_normalised_xsection normalised_xsection['massup'] = massup_normalised_xsection file_template = '{path_to_JSON}/{category}/normalised_xsection_{channel}_{method}.txt' filename = file_template.format( path_to_JSON = path_to_JSON, category = category, channel = channel, method = method, ) if normalise_to_one: filename = filename.replace( 'normalised_xsection', 'normalised_to_one_xsection' ) write_data_to_JSON( normalised_xsection, filename )
def write_fit_results( channel, category, fit_results ): global variable, met_type, output_path folder_template = '%(path)s/%(fit_vars)s/%(CoM)dTeV/%(variable)s/fit_results/%(category)s/' inputs = { 'path':output_path, 'CoM': measurement_config.centre_of_mass_energy, 'variable': variable, 'category': category, 'fit_vars': '_'.join( fit_variables ) } output_folder = folder_template % inputs write_data_to_JSON( fit_results, output_folder + 'fit_results_' + channel + '_' + met_type + '.txt' )
def save(self, output_path): if not self.have_normalisation: self.calculate_normalisation() folder_template = '{path}/normalisation/{method}/{CoM}TeV/{variable}/' folder_template += '{phase_space}/{category}/' inputs = { 'path': output_path, 'CoM': self.config.centre_of_mass_energy, 'variable': self.variable, 'category': self.category, 'method': self.method_string(), 'phase_space': self.phase_space, } output_folder = folder_template.format(**inputs) file_template = '{type}_{channel}_{met_type}.txt' inputs = { 'channel': self.channel, 'met_type': self.met_type, } write_data_to_JSON( self.normalisation, output_folder + file_template.format(type='normalisation', **inputs)) write_data_to_JSON( self.initial_normalisation, output_folder + file_template.format(type='initial_normalisation', **inputs)) write_data_to_JSON( self.templates, output_folder + file_template.format(type='templates', **inputs)) write_data_to_JSON( self.auxiliary_info, output_folder + file_template.format(type='auxiliary_info', **inputs)) return output_folder
def get_unfolded_normalisation(TTJet_fit_results, category, channel): global variable, met_type, path_to_JSON h_truth, h_measured, h_response = get_unfold_histogram_tuple( file_for_unfolding, variable, channel, met_type) MADGRAPH_results = hist_to_value_error_tuplelist(h_truth) POWHEG_results = hist_to_value_error_tuplelist( get_unfold_histogram_tuple(file_for_powheg, variable, channel, met_type)[0]) MCATNLO_results = hist_to_value_error_tuplelist( get_unfold_histogram_tuple(file_for_mcatnlo, variable, channel, met_type)[0]) matchingdown_results = hist_to_value_error_tuplelist( get_unfold_histogram_tuple(file_for_matchingdown, variable, channel, met_type)[0]) matchingup_results = hist_to_value_error_tuplelist( get_unfold_histogram_tuple(file_for_matchingup, variable, channel, met_type)[0]) scaledown_results = hist_to_value_error_tuplelist( get_unfold_histogram_tuple(file_for_scaledown, variable, channel, met_type)[0]) scaleup_results = hist_to_value_error_tuplelist( get_unfold_histogram_tuple(file_for_scaleup, variable, channel, met_type)[0]) TTJet_fit_results_unfolded = unfold_results( TTJet_fit_results, category, channel, h_truth, h_measured, h_response, 'RooUnfoldSvd' # 'TSVDUnfold' ) normalisation_unfolded = { 'TTJet_measured': TTJet_fit_results, 'TTJet_unfolded': TTJet_fit_results_unfolded, 'MADGRAPH': MADGRAPH_results, #other generators 'POWHEG': POWHEG_results, 'MCATNLO': MCATNLO_results, #systematics 'matchingdown': matchingdown_results, 'matchingup': matchingup_results, 'scaledown': scaledown_results, 'scaleup': scaleup_results } write_data_to_JSON( normalisation_unfolded, path_to_JSON + '/' + variable + '/xsection_measurement_results' + '/kv' + str(unfoldCfg.SVD_k_value) + '/' + category + '/normalisation_' + channel + '_' + met_type + '.txt') return normalisation_unfolded
def save(self, output_path): if not self.have_normalisation: self.calculate_normalisation() folder_template = '{path}/normalisation/{method}/{CoM}TeV/{variable}/' folder_template += '{phase_space}/{category}/' inputs = { 'path': output_path, 'CoM': self.config.centre_of_mass_energy, 'variable': self.variable, 'category': self.category, 'method': self.method_string(), 'phase_space': self.phase_space, } output_folder = folder_template.format(**inputs) file_template = '{type}_{channel}_{met_type}.txt' inputs = { 'channel': self.channel, 'met_type': self.met_type, } write_data_to_JSON(self.normalisation, output_folder + file_template.format(type='normalisation', **inputs)) write_data_to_JSON(self.initial_normalisation, output_folder + file_template.format(type='initial_normalisation', **inputs)) write_data_to_JSON(self.templates, output_folder + file_template.format(type='templates', **inputs)) write_data_to_JSON(self.auxiliary_info, output_folder + file_template.format(type='auxiliary_info', **inputs)) return output_folder
def write_normalised_xsection_measurement( measurement, measurement_unfolded, channel, summary = '' ): global path_to_JSON, method output_file = '{path_to_JSON}/central/normalised_xsection_{channel}_{method}_with_errors.txt' output_file = output_file.format( path_to_JSON = path_to_JSON, channel = channel, method = method, ) if not summary == '': output_file = output_file.replace( 'with_errors', summary + '_errors' ) output = {'TTJet_measured':measurement, 'TTJet_unfolded': measurement_unfolded} write_data_to_JSON( output, output_file )
def write_fit_results(channel, category, fit_results): global variable, met_type, output_path folder_template = '%(path)s/%(fit_vars)s/%(CoM)dTeV/%(variable)s/%(category)s/' inputs = { 'path': output_path, 'CoM': measurement_config.centre_of_mass_energy, 'variable': variable, 'category': category, 'fit_vars': '_'.join(fit_variables) } output_folder = folder_template % inputs write_data_to_JSON( fit_results, output_folder + 'normalisation_' + channel + '_' + met_type + '.txt')
def calculate_xsections( normalisation, category, channel ): global variable, met_type, path_to_JSON # calculate the x-sections branching_ratio = 0.15 if channel == 'combined': branching_ratio = branching_ratio * 2 TTJet_xsection = calculate_xsection( normalisation['TTJet_measured'], luminosity, branching_ratio ) # L in pb1 TTJet_withoutFakes_xsection = calculate_xsection( normalisation['TTJet_measured_withoutFakes'], luminosity, branching_ratio ) # L in pb1 TTJet_xsection_unfolded = calculate_xsection( normalisation['TTJet_unfolded'], luminosity, branching_ratio ) # L in pb1 xsection_unfolded = {'TTJet_measured' : TTJet_xsection, 'TTJet_measured_withoutFakes' : TTJet_withoutFakes_xsection, 'TTJet_unfolded' : TTJet_xsection_unfolded, } if category == 'central': powhegPythia8_xsection = calculate_xsection( normalisation['powhegPythia8'], luminosity, branching_ratio ) # L in pb1 amcatnlo_xsection = calculate_xsection( normalisation['amcatnlo'], luminosity, branching_ratio ) # L in pb1 powhegHerwig_xsection = calculate_xsection( normalisation['powhegHerwig'], luminosity, branching_ratio ) # L in pb1 amcatnloHerwig_xsection = calculate_xsection( normalisation['amcatnloHerwig'], luminosity, branching_ratio ) # L in pb1 madgraphMLM_xsection = calculate_xsection( normalisation['madgraphMLM'], luminosity, branching_ratio ) scaledown_xsection = calculate_xsection( normalisation['scaledown'], luminosity, branching_ratio ) # L in pb1 scaleup_xsection = calculate_xsection( normalisation['scaleup'], luminosity, branching_ratio ) # L in pb1 massdown_xsection = calculate_xsection( normalisation['massdown'], luminosity, branching_ratio ) # L in pb1 massup_xsection = calculate_xsection( normalisation['massup'], luminosity, branching_ratio ) # L in pb1 xsection_unfolded['powhegPythia8'] = powhegPythia8_xsection xsection_unfolded['amcatnlo'] = amcatnlo_xsection xsection_unfolded['madgraphMLM'] = madgraphMLM_xsection xsection_unfolded['powhegHerwig'] = powhegHerwig_xsection xsection_unfolded['amcatnloHerwig'] = amcatnloHerwig_xsection xsection_unfolded['scaledown'] = scaledown_xsection xsection_unfolded['scaleup'] = scaleup_xsection xsection_unfolded['massdown'] = massdown_xsection xsection_unfolded['massup'] = massup_xsection file_template = '{path_to_JSON}/{category}/xsection_{channel}_{method}.txt' filename = file_template.format( path_to_JSON = path_to_JSON, category = category, channel = channel, method = method, ) write_data_to_JSON( xsection_unfolded, filename )
def calculate_normalised_xsections(normalisation, category, channel, normalise_to_one=False): global variable, met_type, path_to_JSON TTJet_normalised_xsection = calculate_normalised_xsection( normalisation['TTJet_measured'], bin_widths[variable], normalise_to_one) TTJet_normalised_xsection_unfolded = calculate_normalised_xsection( normalisation['TTJet_unfolded'], bin_widths[variable], normalise_to_one) MADGRAPH_normalised_xsection = calculate_normalised_xsection( normalisation['MADGRAPH'], bin_widths[variable], normalise_to_one) POWHEG_normalised_xsection = calculate_normalised_xsection( normalisation['POWHEG'], bin_widths[variable], normalise_to_one) MCATNLO_normalised_xsection = calculate_normalised_xsection( normalisation['MCATNLO'], bin_widths[variable], normalise_to_one) matchingdown_normalised_xsection = calculate_normalised_xsection( normalisation['matchingdown'], bin_widths[variable], normalise_to_one) matchingup_normalised_xsection = calculate_normalised_xsection( normalisation['matchingup'], bin_widths[variable], normalise_to_one) scaledown_normalised_xsection = calculate_normalised_xsection( normalisation['scaledown'], bin_widths[variable], normalise_to_one) scaleup_normalised_xsection = calculate_normalised_xsection( normalisation['scaleup'], bin_widths[variable], normalise_to_one) normalised_xsection = { 'TTJet_measured': TTJet_normalised_xsection, 'TTJet_unfolded': TTJet_normalised_xsection_unfolded, 'MADGRAPH': MADGRAPH_normalised_xsection, 'POWHEG': POWHEG_normalised_xsection, 'MCATNLO': MCATNLO_normalised_xsection, #systematics 'matchingdown': matchingdown_normalised_xsection, 'matchingup': matchingup_normalised_xsection, 'scaledown': scaledown_normalised_xsection, 'scaleup': scaleup_normalised_xsection } filename = path_to_JSON + '/' + variable + '/xsection_measurement_results' + '/kv' + str( unfoldCfg.SVD_k_value ) + '/' + category + '/normalised_xsection_' + channel + '_' + met_type + '.txt' if normalise_to_one: filename = filename.replace('normalised_xsection', 'normalised_to_one_xsection') write_data_to_JSON(normalised_xsection, filename)
def main(): global config, options parser = OptionParser() parser.add_option( "-p", "--path", dest = "path", default = 'data/fit_checks/no_merging', help = "set path to JSON files" ) parser.add_option( '--create_fit_data', dest = "create_fit_data", action = "store_true", help = "create the fit data for testing." ) parser.add_option( '--refit', dest = "refit", action = "store_true", help = "Fit again even if the output already exists" ) parser.add_option( '--test', dest = "test", action = "store_true", help = "Test only: run just one selected sample" ) variables = config.histogram_path_templates.keys() fit_variables = fit_var_inputs mc_samples = ['TTJet', 'SingleTop', 'QCD', 'V+Jets'] tests = closure_tests channels = ['electron', 'muon'] COMEnergies = ['7', '8'] ( options, _ ) = parser.parse_args() print 'Running from path', options.path if ( options.create_fit_data ): create_fit_data( options.path, variables, fit_variables, mc_samples, COMEnergies = COMEnergies, channels = channels ) output_file = options.path + '/fit_test_output.txt' if options.test: output_file = options.path + '/fit_test_output_test.txt' if options.refit or not os.path.isfile( output_file ) or options.test: if os.path.isfile( options.path + '/fit_check_data.txt' ): fit_data = read_data_from_JSON( options.path + '/fit_check_data.txt' ) results = run_tests( fit_data, COMEnergies, variables, fit_variables, mc_samples, channels, tests ) write_data_to_JSON(results, output_file ) else: print 'Please run bin/prepare_data_for_fit_checks first' print 'Then run this script with the option --create_fit_data.' results = read_data_from_JSON(output_file) plot_results( results )
def calculate_xsections( normalisation, category, channel, k_value = None ): global variable, met_type, path_to_JSON # calculate the x-sections branching_ratio = 0.15 if channel == 'combined': branching_ratio = branching_ratio * 2 TTJet_xsection = calculate_xsection( normalisation['TTJet_measured'], luminosity, branching_ratio ) # L in pb1 TTJet_xsection_unfolded = calculate_xsection( normalisation['TTJet_unfolded'], luminosity, branching_ratio ) # L in pb1 MADGRAPH_xsection = calculate_xsection( normalisation['MADGRAPH'], luminosity, branching_ratio ) # L in pb1 MADGRAPH_ptreweight_xsection = calculate_xsection( normalisation['MADGRAPH_ptreweight'], luminosity, branching_ratio ) # L in pb1 POWHEG_PYTHIA_xsection = calculate_xsection( normalisation['POWHEG_PYTHIA'], luminosity, branching_ratio ) # L in pb1 POWHEG_HERWIG_xsection = calculate_xsection( normalisation['POWHEG_HERWIG'], luminosity, branching_ratio ) # L in pb1 MCATNLO_xsection = None if centre_of_mass == 8: MCATNLO_xsection = calculate_xsection( normalisation['MCATNLO'], luminosity, branching_ratio ) # L in pb1 matchingdown_xsection = calculate_xsection( normalisation['matchingdown'], luminosity, branching_ratio ) # L in pb1 matchingup_xsection = calculate_xsection( normalisation['matchingup'], luminosity, branching_ratio ) # L in pb1 scaledown_xsection = calculate_xsection( normalisation['scaledown'], luminosity, branching_ratio ) # L in pb1 scaleup_xsection = calculate_xsection( normalisation['scaleup'], luminosity, branching_ratio ) # L in pb1 xsection_unfolded = {'TTJet_measured' : TTJet_xsection, 'TTJet_unfolded' : TTJet_xsection_unfolded, 'MADGRAPH': MADGRAPH_xsection, 'MADGRAPH_ptreweight': MADGRAPH_ptreweight_xsection, 'POWHEG_PYTHIA': POWHEG_PYTHIA_xsection, 'POWHEG_HERWIG': POWHEG_HERWIG_xsection, # systematics 'matchingdown': matchingdown_xsection, 'matchingup': matchingup_xsection, 'scaledown': scaledown_xsection, 'scaleup': scaleup_xsection } if centre_of_mass == 8: xsection_unfolded['MCATNLO'] = MCATNLO_xsection if k_value: filename = path_to_JSON + '/xsection_measurement_results/%s/kv%d/%s/xsection_%s.txt' % ( channel, k_value, category, met_type ) elif not channel == 'combined': raise ValueError( 'Invalid k-value for variable %s, channel %s, category %s.' % ( variable, channel, category ) ) else: filename = path_to_JSON + '/xsection_measurement_results/%s/%s/xsection_%s.txt' % ( channel, category, met_type ) write_data_to_JSON( xsection_unfolded, filename )
def write_normalised_xsection_measurement(measurement, measurement_unfolded, channel, summary=''): global path_to_JSON, method output_file = '{path_to_JSON}/central/normalised_xsection_{channel}_{method}_with_errors.txt' output_file = output_file.format( path_to_JSON=path_to_JSON, channel=channel, method=method, ) if not summary == '': output_file = output_file.replace('with_errors', summary + '_errors') output = { 'TTJet_measured': measurement, 'TTJet_unfolded': measurement_unfolded } write_data_to_JSON(output, output_file)
def write_fit_results_and_initial_values(channel, category, fit_results, initial_values, templates): global variable, met_type, output_path output_folder = output_path + '/' + str(measurement_config.centre_of_mass) + 'TeV/' + variable + '/fit_results/' + category + '/' write_data_to_JSON(fit_results, output_folder + 'fit_results_' + channel + '_' + met_type + '.txt') write_data_to_JSON(initial_values, output_folder + 'initial_values_' + channel + '_' + met_type + '.txt') write_data_to_JSON(templates, output_folder + 'templates_' + channel + '_' + met_type + '.txt')
def calculate_normalised_xsections( normalisation, category, channel, k_value = None, normalise_to_one = False ): global variable, met_type, path_to_JSON TTJet_normalised_xsection = calculate_normalised_xsection( normalisation['TTJet_measured'], bin_widths[variable], normalise_to_one ) TTJet_normalised_xsection_unfolded = calculate_normalised_xsection( normalisation['TTJet_unfolded'], bin_widths[variable], normalise_to_one ) MADGRAPH_normalised_xsection = calculate_normalised_xsection( normalisation['MADGRAPH'], bin_widths[variable], normalise_to_one ) MADGRAPH_ptreweight_normalised_xsection = calculate_normalised_xsection( normalisation['MADGRAPH_ptreweight'], bin_widths[variable], normalise_to_one ) POWHEG_PYTHIA_normalised_xsection = calculate_normalised_xsection( normalisation['POWHEG_PYTHIA'], bin_widths[variable], normalise_to_one ) POWHEG_HERWIG_normalised_xsection = calculate_normalised_xsection( normalisation['POWHEG_HERWIG'], bin_widths[variable], normalise_to_one ) MCATNLO_normalised_xsection = None if centre_of_mass == 8: MCATNLO_normalised_xsection = calculate_normalised_xsection( normalisation['MCATNLO'], bin_widths[variable], normalise_to_one ) matchingdown_normalised_xsection = calculate_normalised_xsection( normalisation['matchingdown'], bin_widths[variable], normalise_to_one ) matchingup_normalised_xsection = calculate_normalised_xsection( normalisation['matchingup'], bin_widths[variable], normalise_to_one ) scaledown_normalised_xsection = calculate_normalised_xsection( normalisation['scaledown'], bin_widths[variable], normalise_to_one ) scaleup_normalised_xsection = calculate_normalised_xsection( normalisation['scaleup'], bin_widths[variable], normalise_to_one ) normalised_xsection = {'TTJet_measured' : TTJet_normalised_xsection, 'TTJet_unfolded' : TTJet_normalised_xsection_unfolded, 'MADGRAPH': MADGRAPH_normalised_xsection, 'MADGRAPH_ptreweight': MADGRAPH_ptreweight_normalised_xsection, 'POWHEG_PYTHIA': POWHEG_PYTHIA_normalised_xsection, 'POWHEG_HERWIG': POWHEG_HERWIG_normalised_xsection, # systematics 'matchingdown': matchingdown_normalised_xsection, 'matchingup': matchingup_normalised_xsection, 'scaledown': scaledown_normalised_xsection, 'scaleup': scaleup_normalised_xsection } if centre_of_mass == 8: normalised_xsection['MCATNLO'] = MCATNLO_normalised_xsection if not channel == 'combined': filename = path_to_JSON + '/xsection_measurement_results/%s/kv%d/%s/normalised_xsection_%s.txt' % ( channel, k_value, category, met_type ) else: filename = path_to_JSON + '/xsection_measurement_results/%s/%s/normalised_xsection_%s.txt' % ( channel, category, met_type ) if normalise_to_one: filename = filename.replace( 'normalised_xsection', 'normalised_to_one_xsection' ) write_data_to_JSON( normalised_xsection, filename )
def calculate_xsections(normalisation, category, channel): global variable, met_type, path_to_JSON # calculate the x-sections TTJet_xsection = calculate_xsection(normalisation['TTJet_measured'], luminosity, 0.15) # L in pb1 TTJet_xsection_unfolded = calculate_xsection( normalisation['TTJet_unfolded'], luminosity, 0.15) # L in pb1 MADGRAPH_xsection = calculate_xsection(normalisation['MADGRAPH'], luminosity, 0.15) # L in pb1 POWHEG_xsection = calculate_xsection(normalisation['POWHEG'], luminosity, 0.15) # L in pb1 MCATNLO_xsection = calculate_xsection(normalisation['MCATNLO'], luminosity, 0.15) # L in pb1 matchingdown_xsection = calculate_xsection(normalisation['matchingdown'], luminosity, 0.15) # L in pb1 matchingup_xsection = calculate_xsection(normalisation['matchingup'], luminosity, 0.15) # L in pb1 scaledown_xsection = calculate_xsection(normalisation['scaledown'], luminosity, 0.15) # L in pb1 scaleup_xsection = calculate_xsection(normalisation['scaleup'], luminosity, 0.15) # L in pb1 xsection_unfolded = { 'TTJet_measured': TTJet_xsection, 'TTJet_unfolded': TTJet_xsection_unfolded, 'MADGRAPH': MADGRAPH_xsection, 'POWHEG': POWHEG_xsection, 'MCATNLO': MCATNLO_xsection, #systematics 'matchingdown': matchingdown_xsection, 'matchingup': matchingup_xsection, 'scaledown': scaledown_xsection, 'scaleup': scaleup_xsection } write_data_to_JSON( xsection_unfolded, path_to_JSON + '/' + variable + '/xsection_measurement_results' + '/kv' + str(unfoldCfg.SVD_k_value) + '/' + category + '/xsection_' + channel + '_' + met_type + '.txt')
def get_unfolded_normalisation(TTJet_fit_results, category, channel): global variable, met_type, path_to_JSON h_truth, h_measured, h_response = get_unfold_histogram_tuple(file_for_unfolding, variable, channel, met_type) MADGRAPH_results = hist_to_value_error_tuplelist(h_truth) POWHEG_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_powheg, variable, channel, met_type)[0]) MCATNLO_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_mcatnlo, variable, channel, met_type)[0]) matchingdown_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_matchingdown, variable, channel, met_type)[0]) matchingup_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_matchingup, variable, channel, met_type)[0]) scaledown_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_scaledown, variable, channel, met_type)[0]) scaleup_results = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_scaleup, variable, channel, met_type)[0]) TTJet_fit_results_unfolded = unfold_results(TTJet_fit_results, category, channel, h_truth, h_measured, h_response, 'RooUnfoldSvd' # 'TSVDUnfold' ) normalisation_unfolded = { 'TTJet_measured' : TTJet_fit_results, 'TTJet_unfolded' : TTJet_fit_results_unfolded, 'MADGRAPH': MADGRAPH_results, #other generators 'POWHEG': POWHEG_results, 'MCATNLO': MCATNLO_results, #systematics 'matchingdown': matchingdown_results, 'matchingup': matchingup_results, 'scaledown': scaledown_results, 'scaleup': scaleup_results } write_data_to_JSON(normalisation_unfolded, path_to_JSON + '/' + variable + '/xsection_measurement_results' + '/kv' + str(unfoldCfg.SVD_k_value) + '/' + category + '/normalisation_' + channel + '_' + met_type + '.txt') return normalisation_unfolded
luminosity = 5050 # L in pb1 branching_ratio = 0.15 # to be corrected for normalisation_electron_unfolded = read_data_from_JSON(csm.normalisation_electron_unfolded_file) normalisation_muon_unfolded = read_data_from_JSON(csm.normalisation_muon_unfolded_file) normalisation_combined_unfolded = read_data_from_JSON(csm.normalisation_combined_unfolded_file) xsection_electron_unfolded = get_cross_sections(normalisation_electron_unfolded, luminosity, branching_ratio) xsection_muon_unfolded = get_cross_sections(normalisation_muon_unfolded, luminosity, branching_ratio) xsection_combined_unfolded = get_cross_sections(normalisation_combined_unfolded, luminosity, branching_ratio*2) normalised_to_one_xsection_electron = get_normalised_xsection(normalisation_electron_unfolded, True) normalised_to_one_xsection_muon = get_normalised_xsection(normalisation_muon_unfolded, True) normalised_to_one_xsection_combined = get_normalised_xsection(normalisation_combined_unfolded, True) normalised_xsection_electron = get_normalised_xsection(normalisation_electron_unfolded, False) normalised_xsection_muon = get_normalised_xsection(normalisation_muon_unfolded, False) normalised_xsection_combined = get_normalised_xsection(normalisation_combined_unfolded, False) #write results write_data_to_JSON(xsection_electron_unfolded, csm.xsection_electron_unfolded_file) write_data_to_JSON(xsection_muon_unfolded, csm.xsection_muon_unfolded_file) write_data_to_JSON(xsection_combined_unfolded, csm.xsection_combined_unfolded_file) write_data_to_JSON(normalised_to_one_xsection_electron, csm.normalised_to_one_xsection_electron_unfolded_file) write_data_to_JSON(normalised_to_one_xsection_muon, csm.normalised_to_one_xsection_muon_unfolded_file) write_data_to_JSON(normalised_to_one_xsection_combined, csm.normalised_to_one_xsection_combined_unfolded_file) write_data_to_JSON(normalised_xsection_electron, csm.normalised_xsection_electron_unfolded_file) write_data_to_JSON(normalised_xsection_muon, csm.normalised_xsection_muon_unfolded_file) write_data_to_JSON(normalised_xsection_combined, csm.normalised_xsection_combined_unfolded_file)
# ad-hoc switch for PFMET -> patMETsPFlow if met_type == 'PFMET': met_type = 'patMETsPFlow' filename = '' # get unfolded normalisation unfolded_normalisation_electron = get_unfolded_normalisation( TTJet_fit_results_electron, category, 'electron', k_value_electron ) unfolded_normalisation_muon = get_unfolded_normalisation( TTJet_fit_results_muon, category, 'muon', k_value_muon ) if combine_before_unfolding: unfolded_normalisation_combined = get_unfolded_normalisation( TTJet_fit_results_combined, category, 'combined', k_value_combined ) else: unfolded_normalisation_combined = combine_complex_results( unfolded_normalisation_electron, unfolded_normalisation_muon ) filename = path_to_JSON + '/xsection_measurement_results/electron/kv%d/%s/normalisation_%s.txt' % ( k_value_electron, category, met_type ) write_data_to_JSON( unfolded_normalisation_electron, filename ) filename = path_to_JSON + '/xsection_measurement_results/muon/kv%d/%s/normalisation_%s.txt' % ( k_value_muon, category, met_type ) write_data_to_JSON( unfolded_normalisation_muon, filename ) filename = path_to_JSON + '/xsection_measurement_results/combined/%s/normalisation_%s.txt' % ( category, met_type ) write_data_to_JSON( unfolded_normalisation_combined, filename ) if measurement_config.include_higgs: # now the same for the Higgs Higgs_fit_results_electron = fit_results_electron['Higgs'] Higgs_fit_results_muon = fit_results_muon['Higgs'] Higgs_fit_results_combined = fit_results_combined['Higgs'] unfolded_normalisation_electron_higgs = get_unfolded_normalisation( fit_results_electron['Higgs'], category, 'electron', k_value_electron ) unfolded_normalisation_muon_higgs = get_unfolded_normalisation( fit_results_muon['Higgs'], category, 'muon', k_value_muon ) if combine_before_unfolding: unfolded_normalisation_combined_higgs = get_unfolded_normalisation( fit_results_combined['Higgs'], category, 'combined', k_value_combined ) else:
def main(): ''' Step 1: Get the 2D histogram for every sample (channel and/or centre of mass energy) Step 2: Change the size of the first bin until it fulfils the minimal criteria Step 3: Check if it is true for all other histograms. If not back to step 2 Step 4: Repeat step 2 & 3 until no mo bins can be created ''' parser = OptionParser() parser.add_option( '-v', dest = "visiblePhaseSpace", action = "store_true", help = "Consider visible phase space or not" ) parser.add_option( '-c', dest = "combined", action = "store_true", help = "Combine channels" ) ( options, _ ) = parser.parse_args() p_min = 0.6 s_min = 0.6 # we also want the statistical error to be larger than 5% # this translates (error -= 1/sqrt(N)) to (1/0.05)^2 = 400 n_min = 200 n_min_lepton = 500 # n_min = 200 # N = 200 -> 7.1 % stat error bin_choices = {} variables = bin_edges.keys() for variable in variables: print('--- Doing variable',variable) variableToUse = variable if 'Rap' in variable: variableToUse = 'abs_%s' % variable histogram_information = get_histograms( variableToUse, options ) if variable == 'HT': best_binning, histogram_information = get_best_binning( histogram_information , p_min, s_min, n_min, minimum_bin_width[variable], x_min=100. ) elif variable == 'ST': best_binning, histogram_information = get_best_binning( histogram_information , p_min, s_min, n_min, minimum_bin_width[variable], x_min=123. ) elif variable == 'NJets': best_binning, histogram_information = get_best_binning( histogram_information , p_min, s_min, n_min, minimum_bin_width[variable], x_min=3.5 ) elif variable == 'lepton_pt': best_binning, histogram_information = get_best_binning( histogram_information , p_min, s_min, n_min_lepton, minimum_bin_width[variable], x_min=23. ) elif variable == 'abs_lepton_eta': best_binning, histogram_information = get_best_binning( histogram_information , p_min, s_min, n_min_lepton, minimum_bin_width[variable] ) else: best_binning, histogram_information = get_best_binning( histogram_information , p_min, s_min, n_min, minimum_bin_width[variable], ) if 'Rap' in variable: for b in list(best_binning): if b != 0.0: best_binning.append(-1.0*b) best_binning.sort() # Make last bin smaller if huge # Won't change final results if len(best_binning) >= 4: lastBinWidth = best_binning[-1] - best_binning[-2] penultimateBinWidth = best_binning[-2] - best_binning[-3] if lastBinWidth / penultimateBinWidth > 5: newLastBinWidth = penultimateBinWidth * 5 best_binning[-1] = best_binning[-2] + newLastBinWidth if variable == 'abs_lepton_eta': best_binning = [ round(i,2) for i in best_binning ] elif variable != 'NJets' : best_binning = [ round(i) for i in best_binning ] bin_choices[variable] = best_binning print('The best binning for', variable, 'is:') print('bin edges =', best_binning) print('N_bins =', len( best_binning ) - 1) print('The corresponding purities and stabilities are:') for info in histogram_information: # print_latex_table(info, variable, best_binning) outputInfo = {} outputInfo['p_i'] = info['p_i'] outputInfo['s_i'] = info['s_i'] outputInfo['N'] = info['N'] outputJsonFile = 'unfolding/13TeV/binningInfo_%s_%s_FullPS.txt' % ( variable, info['channel'] ) if options.visiblePhaseSpace: outputJsonFile = 'unfolding/13TeV/binningInfo_%s_%s_VisiblePS.txt' % ( variable, info['channel'] ) write_data_to_JSON( outputInfo, outputJsonFile ) for key in outputInfo: print (key,outputInfo[key]) print('-' * 120) print('=' * 120) print('For config/variable_binning.py') print('=' * 120) for variable in bin_choices: print('\''+variable+'\' : '+str(bin_choices[variable])+',')
unfolded_normalisation_muon = {} # Electron channel unfolded_normalisation_electron = get_unfolded_normalisation( TTJet_fit_results_electron, category, 'electron', tau_value_electron, visiblePS=visiblePS) filename = file_template.format( path_to_JSON=path_to_JSON, category=category, channel='electron', method=method, ) write_data_to_JSON(unfolded_normalisation_electron, filename) # measure xsection calculate_xsections(unfolded_normalisation_electron, category, 'electron') calculate_normalised_xsections(unfolded_normalisation_electron, category, 'electron') calculate_normalised_xsections(unfolded_normalisation_electron, category, 'electron', True) # Muon channel unfolded_normalisation_muon = get_unfolded_normalisation( TTJet_fit_results_muon, category, 'muon', tau_value_muon, visiblePS=visiblePS)
def create_fit_data(path, variables, fit_variables, mc_samples, COMEnergies, channels): ''' Creates the fit data in path + fit_check_data.txt in the JSON format. The resulting dictionary is of the form {centre-of-mass-energy: { channel : { variable : { variable_bin : { fit_variable : { templates : { sample : []}, initial_values : { sample : []} } } } } } } ''' # first step is to read the data raw_data = {} # the folder structure is # path/fit_variable/centre-of-mass-energy/variable/fit_results/central/* for fit_variable in fit_variables: raw_data[fit_variable] = {} for COMEnergy in COMEnergies: raw_data[fit_variable][COMEnergy] = {} for variable in variables: raw_data[fit_variable][COMEnergy][variable] = {} for channel in channels: raw_data[fit_variable][COMEnergy][variable][channel] = {} data_path = path + '/' + fit_variable + '/' + str( COMEnergy) + 'TeV' templates = read_fit_templates(data_path, variable, channel=channel) initial_values = read_initial_normalisation( data_path, variable, channel=channel) raw_data[fit_variable][COMEnergy][variable][channel][ 'templates'] = templates raw_data[fit_variable][COMEnergy][variable][channel][ 'initial_values'] = initial_values # put it into the new structure fit_data = {} for COMEnergy in COMEnergies: fit_data[COMEnergy] = {} for channel in channels: fit_data[COMEnergy][channel] = {} for variable in variables: fit_data[COMEnergy][channel][variable] = {} for v_bin in variable_bins_ROOT[variable]: fit_data[COMEnergy][channel][variable][v_bin] = {} for fit_variable in fit_variables: fit_data[COMEnergy][channel][variable][v_bin][ fit_variable] = {} fit_data[COMEnergy][channel][variable][v_bin][ fit_variable]['templates'] = raw_data[ fit_variable][COMEnergy][variable][channel][ 'templates'] fit_data[COMEnergy][channel][variable][v_bin][ fit_variable]['initial_values'] = raw_data[ fit_variable][COMEnergy][variable][channel][ 'initial_values'] write_data_to_JSON(fit_data, path + '/fit_check_data.txt', indent=False)
def main(): """ Step 1: Get the 2D histogram for every sample (channel and/or centre of mass energy) Step 2: Change the size of the first bin until it fulfils the minimal criteria Step 3: Check if it is true for all other histograms. If not back to step 2 Step 4: Repeat step 2 & 3 until no mo bins can be created """ parser = OptionParser() parser.add_option("-v", dest="visiblePhaseSpace", action="store_true", help="Consider visible phase space or not") parser.add_option("-c", dest="combined", action="store_true", help="Combine channels") (options, _) = parser.parse_args() measurement_config = XSectionConfig(13) p_min = 0.6 # 0.5 for MET s_min = 0.6 # we also want the statistical error to be larger than 5% # this translates (error -= 1/sqrt(N)) to (1/0.05)^2 = 400 n_min = 500 n_min_lepton = 500 # n_min = 200 # N = 200 -> 7.1 % stat error bin_choices = {} # variables = bin_edges_full.keys() variables = measurement_config.variables for variable in variables: global var var = variable print("--- Doing variable", variable) variableToUse = variable if "Rap" in variable: variableToUse = "abs_%s" % variable histogram_information = get_histograms(variableToUse, options) if variable == "HT": best_binning, histogram_information = get_best_binning( histogram_information, p_min, s_min, n_min, minimum_bin_width[variable], x_min=100.0 ) elif variable == "ST": best_binning, histogram_information = get_best_binning( histogram_information, p_min, s_min, n_min, minimum_bin_width[variable], x_min=123.0 ) elif variable == "MET": best_binning, histogram_information = get_best_binning( histogram_information, 0.5, 0.5, n_min, minimum_bin_width[variable] ) elif variable == "NJets": best_binning, histogram_information = get_best_binning( histogram_information, p_min, s_min, n_min, minimum_bin_width[variable], x_min=3.5 ) elif variable == "lepton_pt": best_binning, histogram_information = get_best_binning( histogram_information, p_min, s_min, n_min_lepton, minimum_bin_width[variable], x_min=23.0 ) elif variable == "abs_lepton_eta": best_binning, histogram_information = get_best_binning( histogram_information, p_min, s_min, n_min_lepton, minimum_bin_width[variable] ) else: best_binning, histogram_information = get_best_binning( histogram_information, p_min, s_min, n_min, minimum_bin_width[variable] ) if "Rap" in variable: for b in list(best_binning): if b != 0.0: best_binning.append(-1.0 * b) best_binning.sort() # Make last bin smaller if huge # Won't change final results if len(best_binning) >= 4: lastBinWidth = best_binning[-1] - best_binning[-2] penultimateBinWidth = best_binning[-2] - best_binning[-3] if lastBinWidth / penultimateBinWidth > 5: newLastBinWidth = penultimateBinWidth * 5 best_binning[-1] = best_binning[-2] + newLastBinWidth if variable == "abs_lepton_eta": best_binning = [round(i, 2) for i in best_binning] elif variable != "NJets": best_binning = [round(i) for i in best_binning] bin_choices[variable] = best_binning print("The best binning for", variable, "is:") print("bin edges =", best_binning) print("N_bins =", len(best_binning) - 1) print("The corresponding purities and stabilities are:") for info in histogram_information: # print_latex_table(info, variable, best_binning) outputInfo = {} outputInfo["p_i"] = info["p_i"] outputInfo["s_i"] = info["s_i"] outputInfo["N"] = info["N"] outputJsonFile = "unfolding/13TeV/binningInfo_%s_%s_FullPS.txt" % (variable, info["channel"]) if options.visiblePhaseSpace: outputJsonFile = "unfolding/13TeV/binningInfo_%s_%s_VisiblePS.txt" % (variable, info["channel"]) write_data_to_JSON(outputInfo, outputJsonFile) for key in outputInfo: print(key, outputInfo[key]) print("-" * 120) print("=" * 120) print("For config/variable_binning.py") print("=" * 120) for variable in bin_choices: print("'" + variable + "' : " + str(bin_choices[variable]) + ",")
def save_pulls( pulls, test, method, channel ): global use_N_toy, offset_toy_mc, offset_toy_data file_template = 'Pulls_%s_%s_%s_toy_MC_%d_to_%d_MC_%d_to_%d_data.txt' output_file = output_folder + file_template % ( test, method, channel, offset_toy_mc + 1, use_N_toy + offset_toy_mc, offset_toy_data + 1, use_N_toy + offset_toy_data ) write_data_to_JSON( pulls, output_file ) print 'Pulls saved in file: ', output_file
def write_fit_results_and_initial_values(fit_results_electron, fit_results_muon, initial_values_electron, initial_values_muon): write_data_to_JSON(fit_results_electron, 'data/fit_results_electron.txt') write_data_to_JSON(fit_results_muon, 'data/fit_results_muon.txt') write_data_to_JSON(initial_values_electron, 'data/initial_values_electron.txt') write_data_to_JSON(initial_values_muon, 'data/initial_values_muon.txt')
def calculate_xsections(normalisation, category, channel): global variable, met_type, path_to_JSON # calculate the x-sections branching_ratio = 0.15 if channel == 'combined': branching_ratio = branching_ratio * 2 TTJet_xsection = calculate_xsection(normalisation['TTJet_measured'], luminosity, branching_ratio) # L in pb1 TTJet_withoutFakes_xsection = calculate_xsection( normalisation['TTJet_measured_withoutFakes'], luminosity, branching_ratio) # L in pb1 TTJet_xsection_unfolded = calculate_xsection( normalisation['TTJet_unfolded'], luminosity, branching_ratio) # L in pb1 xsection_unfolded = { 'TTJet_measured': TTJet_xsection, 'TTJet_measured_withoutFakes': TTJet_withoutFakes_xsection, 'TTJet_unfolded': TTJet_xsection_unfolded, } if category == 'central': powhegPythia8_xsection = calculate_xsection( normalisation['powhegPythia8'], luminosity, branching_ratio) # L in pb1 amcatnlo_xsection = calculate_xsection(normalisation['amcatnlo'], luminosity, branching_ratio) # L in pb1 amcatnlo_HERWIG_xsection = calculate_xsection( normalisation['amcatnlo_HERWIG'], luminosity, branching_ratio) # L in pb1 scaledown_xsection = calculate_xsection(normalisation['scaledown'], luminosity, branching_ratio) # L in pb1 scaleup_xsection = calculate_xsection(normalisation['scaleup'], luminosity, branching_ratio) # L in pb1 massdown_xsection = calculate_xsection(normalisation['massdown'], luminosity, branching_ratio) # L in pb1 massup_xsection = calculate_xsection(normalisation['massup'], luminosity, branching_ratio) # L in pb1 madgraphMLM_xsection = calculate_xsection(normalisation['madgraphMLM'], luminosity, branching_ratio) xsection_unfolded['powhegPythia8'] = powhegPythia8_xsection xsection_unfolded['amcatnlo'] = amcatnlo_xsection xsection_unfolded['madgraphMLM'] = madgraphMLM_xsection xsection_unfolded['amcatnlo_HERWIG'] = amcatnlo_HERWIG_xsection xsection_unfolded['scaledown'] = scaledown_xsection xsection_unfolded['scaleup'] = scaleup_xsection xsection_unfolded['massdown'] = massdown_xsection xsection_unfolded['massup'] = massup_xsection file_template = '{path_to_JSON}/{category}/xsection_{channel}_{method}.txt' filename = file_template.format( path_to_JSON=path_to_JSON, category=category, channel=channel, method=method, ) write_data_to_JSON(xsection_unfolded, filename)
def save_pulls(pulls, test, method, channel): global use_N_toy, skip_N_toy file_template = "Pulls_%s_%s_%s_toy_MC_%d_to_%d.txt" output_file = output_folder + file_template % (test, method, channel, skip_N_toy + 1, use_N_toy + skip_N_toy) write_data_to_JSON(pulls, output_file)
def toJSON(self, JSON_file): output = self.toDict() filename = JSON_file.split('/')[-1] directory = JSON_file.replace(filename, '') fu.make_folder_if_not_exists(directory) fu.write_data_to_JSON(output, JSON_file)
def calculate_normalised_xsections(normalisation, category, channel, normalise_to_one=False): global variable, met_type, path_to_JSON, phase_space binWidths = None if phase_space == 'VisiblePS': binWidths = bin_widths_visiblePS elif phase_space == 'FullPS': binWidths = bin_widths TTJet_normalised_xsection = calculate_normalised_xsection( normalisation['TTJet_measured'], binWidths[variable], normalise_to_one) TTJet_withoutFakes_normalised_xsection = calculate_normalised_xsection( normalisation['TTJet_measured_withoutFakes'], binWidths[variable], normalise_to_one) TTJet_normalised_xsection_unfolded = calculate_normalised_xsection( normalisation['TTJet_unfolded'], binWidths[variable], normalise_to_one) normalised_xsection = { 'TTJet_measured': TTJet_normalised_xsection, 'TTJet_measured_withoutFakes': TTJet_withoutFakes_normalised_xsection, 'TTJet_unfolded': TTJet_normalised_xsection_unfolded } if category == 'central': powhegPythia8_normalised_xsection = calculate_normalised_xsection( normalisation['powhegPythia8'], binWidths[variable], normalise_to_one) amcatnlo_normalised_xsection = calculate_normalised_xsection( normalisation['amcatnlo'], binWidths[variable], normalise_to_one) amcatnlo_HERWIG_normalised_xsection = calculate_normalised_xsection( normalisation['amcatnlo_HERWIG'], binWidths[variable], normalise_to_one) scaledown_normalised_xsection = calculate_normalised_xsection( normalisation['scaledown'], binWidths[variable], normalise_to_one) scaleup_normalised_xsection = calculate_normalised_xsection( normalisation['scaleup'], binWidths[variable], normalise_to_one) massdown_normalised_xsection = calculate_normalised_xsection( normalisation['massdown'], binWidths[variable], normalise_to_one) massup_normalised_xsection = calculate_normalised_xsection( normalisation['massup'], binWidths[variable], normalise_to_one) madgraphMLM_normalised_xsection = calculate_normalised_xsection( normalisation['madgraphMLM'], binWidths[variable], normalise_to_one) normalised_xsection[ 'powhegPythia8'] = powhegPythia8_normalised_xsection normalised_xsection['amcatnlo'] = amcatnlo_normalised_xsection normalised_xsection['madgraphMLM'] = madgraphMLM_normalised_xsection normalised_xsection[ 'amcatnlo_HERWIG'] = amcatnlo_HERWIG_normalised_xsection normalised_xsection['scaledown'] = scaledown_normalised_xsection normalised_xsection['scaleup'] = scaleup_normalised_xsection normalised_xsection['massdown'] = massdown_normalised_xsection normalised_xsection['massup'] = massup_normalised_xsection file_template = '{path_to_JSON}/{category}/normalised_xsection_{channel}_{method}.txt' filename = file_template.format( path_to_JSON=path_to_JSON, category=category, channel=channel, method=method, ) if normalise_to_one: filename = filename.replace('normalised_xsection', 'normalised_to_one_xsection') write_data_to_JSON(normalised_xsection, filename)
POWHEG_results_electron = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_powheg, 'electron')[1]) PYTHIA_results_electron = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_pythia, 'electron')[1]) MCATNLO_results_electron = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_mcatnlo, 'electron')[1]) matchingdown_results_electron = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_matchingdown, 'electron')[1]) matchingup_results_electron = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_matchingup, 'electron')[1]) scaledown_results_electron = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_scaledown, 'electron')[1]) scaleup_results_electron = hist_to_value_error_tuplelist(get_unfold_histogram_tuple(file_for_scaleup, 'electron')[1]) TTJet_fit_results_electron_unfolded = unfold_results(TTJet_fit_results_electron, h_truth, h_measured, h_response, 'RooUnfoldSvd') write_data_to_JSON(TTJet_fit_results_electron, 'data/TTJet_fit_results_electron.txt') normalisation_electron_unfolded = { 'TTJet' : TTJet_fit_results_electron_unfolded, 'MADGRAPH': MADGRAPH_results_electron, #other generators 'POWHEG': POWHEG_results_electron, 'PYTHIA': PYTHIA_results_electron, 'MCATNLO': MCATNLO_results_electron, #systematics 'matchingdown': matchingdown_results_electron, 'matchingup': matchingup_results_electron, 'scaledown': scaledown_results_electron, 'scaleup': scaleup_results_electron } write_data_to_JSON(normalisation_electron_unfolded, 'data/normalisation_electron_unfolded.txt')
def write_fit_results_and_initial_values(channel, category, fit_results, initial_values, templates): global variable, met_type write_data_to_JSON(fit_results, 'data/' + variable + '/fit_results/' + category + '/fit_results_' + channel + '_' + met_type + '.txt') write_data_to_JSON(initial_values, 'data/' + variable + '/fit_results/' + category + '/initial_values_' + channel + '_' + met_type + '.txt') write_data_to_JSON(templates, 'data/' + variable + '/fit_results/' + category + '/templates_' + channel + '_' + met_type + '.txt')
muon_file = path_to_JSON + '/fit_results/' + ttbar_theory_systematic_prefix + 'mcatnlo' + '/fit_results_muon_' + met_type + '.txt' TTJet_fit_results_electron = read_data_from_JSON(electron_file)['TTJet'] TTJet_fit_results_muon = read_data_from_JSON(muon_file)['TTJet'] # change back to original MET type for the unfolding met_type = translate_options[options.metType] # ad-hoc switch for PFMET -> patMETsPFlow if met_type == 'PFMET': met_type = 'patMETsPFlow' # get unfolded normalisation unfolded_normalisation_electron = get_unfolded_normalisation(TTJet_fit_results_electron, category, 'electron') unfolded_normalisation_muon = get_unfolded_normalisation(TTJet_fit_results_muon, category, 'muon') unfolded_normalisation_combined = combine_complex_results(unfolded_normalisation_electron, unfolded_normalisation_muon) write_data_to_JSON(unfolded_normalisation_combined, path_to_JSON + '/xsection_measurement_results' + '/kv' + str(unfoldCfg.SVD_k_value) + '/' + category + '/normalisation_combined_' + met_type + '.txt') # measure xsection calculate_xsections(unfolded_normalisation_electron, category, 'electron') calculate_xsections(unfolded_normalisation_muon, category, 'muon') calculate_xsections(unfolded_normalisation_combined, category, 'combined') calculate_normalised_xsections(unfolded_normalisation_electron, category, 'electron') calculate_normalised_xsections(unfolded_normalisation_muon, category, 'muon') calculate_normalised_xsections(unfolded_normalisation_combined, category, 'combined') normalise_to_one = True calculate_normalised_xsections(unfolded_normalisation_electron, category, 'electron', normalise_to_one) calculate_normalised_xsections(unfolded_normalisation_muon, category, 'muon', normalise_to_one) calculate_normalised_xsections(unfolded_normalisation_combined, category, 'combined', normalise_to_one)
file_template = '{path_to_JSON}/{category}/unfolded_normalisation_{channel}_{method}.txt' filename = '' # # get unfolded normalisation unfolded_normalisation_electron = {} unfolded_normalisation_muon = {} # Electron channel unfolded_normalisation_electron = get_unfolded_normalisation( TTJet_fit_results_electron, category, 'electron', tau_value_electron, visiblePS = visiblePS ) filename = file_template.format( path_to_JSON = path_to_JSON, category = category, channel = 'electron', method = method, ) write_data_to_JSON( unfolded_normalisation_electron, filename ) # measure xsection calculate_xsections( unfolded_normalisation_electron, category, 'electron' ) calculate_normalised_xsections( unfolded_normalisation_electron, category, 'electron' ) calculate_normalised_xsections( unfolded_normalisation_electron, category, 'electron' , True ) # Muon channel unfolded_normalisation_muon = get_unfolded_normalisation( TTJet_fit_results_muon, category, 'muon', tau_value_muon, visiblePS = visiblePS ) filename = file_template.format( path_to_JSON = path_to_JSON, category = category, channel = 'muon', method = method, ) write_data_to_JSON( unfolded_normalisation_muon, filename ) # measure xsection
def save_pulls(pulls, test, method, channel): global use_N_toy, skip_N_toy file_template = 'Pulls_%s_%s_%s_toy_MC_%d_to_%d.txt' output_file = output_folder + file_template % ( test, method, channel, skip_N_toy + 1, use_N_toy + skip_N_toy) write_data_to_JSON(pulls, output_file)