def read_fit_templates_and_results_as_histograms( category, channel ):
    global path_to_JSON, variable, met_type, phase_space
    templates = read_data_from_JSON( path_to_JSON + '/fit_results/' + category + '/templates_' + channel + '_' + met_type + '.txt' )

    data_values = read_data_from_JSON( path_to_JSON + '/fit_results/' + category + '/initial_values_' + channel + '_' + met_type + '.txt' )['data']
    fit_results = read_data_from_JSON( path_to_JSON + '/fit_results/' + category + '/fit_results_' + channel + '_' + met_type + '.txt' )
    fit_variables = templates.keys()
    template_histograms = {fit_variable: {} for fit_variable in fit_variables}
    fit_results_histograms = {fit_variable: {} for fit_variable in fit_variables}

    variableBins = None
    if phase_space == 'VisiblePS':
        variableBins = variable_bins_visiblePS_ROOT
    elif phase_space == 'FullPS':
        variableBins = variable_bins_ROOT

    for bin_i, variable_bin in enumerate( variableBins[variable] ):
        for fit_variable in fit_variables:
            h_template_data = value_tuplelist_to_hist( templates[fit_variable]['data'][bin_i], fit_variable_bin_edges[fit_variable] )
            h_template_ttjet =  value_tuplelist_to_hist( templates[fit_variable]['TTJet'][bin_i], fit_variable_bin_edges[fit_variable] )
            h_template_singletop =  value_tuplelist_to_hist( templates[fit_variable]['SingleTop'][bin_i], fit_variable_bin_edges[fit_variable] )
            h_template_VJets = value_tuplelist_to_hist( templates[fit_variable]['V+Jets'][bin_i], fit_variable_bin_edges[fit_variable] )
            h_template_QCD = value_tuplelist_to_hist( templates[fit_variable]['QCD'][bin_i], fit_variable_bin_edges[fit_variable] )
            template_histograms[fit_variable][variable_bin] = {
                                        'TTJet' : h_template_ttjet,
                                        'SingleTop' : h_template_singletop,
                                        'V+Jets':h_template_VJets,
                                        'QCD':h_template_QCD
                                        }
            h_data = h_template_data.Clone()
            h_ttjet = h_template_ttjet.Clone()
            h_singletop = h_template_singletop.Clone()
            h_VJets = h_template_VJets.Clone()
            h_QCD = h_template_QCD.Clone()

            data_normalisation = data_values[bin_i][0]
            n_ttjet = fit_results['TTJet'][bin_i][0]
            n_singletop = fit_results['SingleTop'][bin_i][0]
            VJets_normalisation = fit_results['V+Jets'][bin_i][0]
            QCD_normalisation = fit_results['QCD'][bin_i][0]

            h_data.Scale( data_normalisation )
            h_ttjet.Scale( n_ttjet )
            h_singletop.Scale( n_singletop )
            h_VJets.Scale( VJets_normalisation )
            h_QCD.Scale( QCD_normalisation )
            h_background = h_VJets + h_QCD + h_singletop

            for bin_i_data in range( len( h_data ) ):
                h_data.SetBinError( bin_i_data + 1, sqrt( h_data.GetBinContent( bin_i_data + 1 ) ) )

            fit_results_histograms[fit_variable][variable_bin] = {
                                                    'data' : h_data,
                                                    'signal' : h_ttjet,
                                                    'background' : h_background
                                                    }

    return template_histograms, fit_results_histograms
def read_fit_templates_and_results_as_histograms(category, channel):
    global path_to_JSON, variable, met_type
    templates = read_data_from_JSON(path_to_JSON + '/' + variable +
                                    '/fit_results/' + category +
                                    '/templates_' + channel + '_' + met_type +
                                    '.txt')
    data_values = read_data_from_JSON(path_to_JSON + '/' + variable +
                                      '/fit_results/' + category +
                                      '/initial_values_' + channel + '_' +
                                      met_type + '.txt')['data']
    fit_results = read_data_from_JSON(path_to_JSON + '/' + variable +
                                      '/fit_results/' + category +
                                      '/fit_results_' + channel + '_' +
                                      met_type + '.txt')
    template_histograms = {}
    fit_results_histograms = {}
    for bin_i, variable_bin in enumerate(variable_bins_ROOT[variable]):
        h_template_data = value_tuplelist_to_hist(templates['data'][bin_i],
                                                  eta_bin_edges)
        h_template_signal = value_tuplelist_to_hist(templates['signal'][bin_i],
                                                    eta_bin_edges)
        h_template_VJets = value_tuplelist_to_hist(templates['V+Jets'][bin_i],
                                                   eta_bin_edges)
        h_template_QCD = value_tuplelist_to_hist(templates['QCD'][bin_i],
                                                 eta_bin_edges)
        template_histograms[variable_bin] = {
            'signal': h_template_signal,
            'V+Jets': h_template_VJets,
            'QCD': h_template_QCD
        }
        h_data = h_template_data.Clone()
        h_signal = h_template_signal.Clone()
        h_VJets = h_template_VJets.Clone()
        h_QCD = h_template_QCD.Clone()

        data_normalisation = data_values[bin_i]
        signal_normalisation = fit_results['signal'][bin_i][0]
        VJets_normalisation = fit_results['V+Jets'][bin_i][0]
        QCD_normalisation = fit_results['QCD'][bin_i][0]

        h_data.Scale(data_normalisation)
        h_signal.Scale(signal_normalisation)
        h_VJets.Scale(VJets_normalisation)
        h_QCD.Scale(QCD_normalisation)
        h_background = h_VJets + h_QCD

        for bin_i in range(len(h_data)):
            h_data.SetBinError(bin_i + 1,
                               sqrt(h_data.GetBinContent(bin_i + 1)))

        fit_results_histograms[variable_bin] = {
            'data': h_data,
            'signal': h_signal,
            'background': h_background
        }

    return template_histograms, fit_results_histograms
def compare_combine_before_after_unfolding(measurement='normalised_xsection',
                                           add_before_unfolding=False):
    file_template = 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += '{measurement}_{channel}_RooUnfold{method}.txt'

    variables = [
        'MET', 'HT', 'ST', 'NJets', 'lepton_pt', 'abs_lepton_eta', 'WPT'
    ]
    for variable in variables:
        combineBefore = file_template.format(variable=variable,
                                             method='Svd',
                                             channel='combinedBeforeUnfolding',
                                             measurement=measurement)
        combineAfter = file_template.format(variable=variable,
                                            method='Svd',
                                            channel='combined',
                                            measurement=measurement)
        data = read_data_from_JSON(combineBefore)
        before_unfolding = data['TTJet_measured']
        combineBefore_data = data['TTJet_unfolded']
        combineAfter_data = read_data_from_JSON(combineAfter)['TTJet_unfolded']
        h_combineBefore = value_error_tuplelist_to_hist(
            combineBefore_data, bin_edges_vis[variable])
        h_combineAfter = value_error_tuplelist_to_hist(combineAfter_data,
                                                       bin_edges_vis[variable])
        h_before_unfolding = value_error_tuplelist_to_hist(
            before_unfolding, bin_edges_vis[variable])

        properties = Histogram_properties()
        properties.name = '{0}_compare_combine_before_after_unfolding_{1}'.format(
            measurement, variable)
        properties.title = 'Comparison of combining before/after unfolding'
        properties.path = 'plots'
        properties.has_ratio = True
        properties.xerr = True
        properties.x_limits = (bin_edges_vis[variable][0],
                               bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        if 'xsection' in measurement:
            properties.y_axis_title = r'$\frac{1}{\sigma}  \frac{d\sigma}{d' + \
                variables_latex[variable] + '}$'
        else:
            properties.y_axis_title = r'$t\bar{t}$ normalisation'

        histograms = {
            'Combine before unfolding': h_combineBefore,
            'Combine after unfolding': h_combineAfter
        }
        if add_before_unfolding:
            histograms['before unfolding'] = h_before_unfolding
            properties.name += '_ext'
            properties.has_ratio = False
        plot = Plot(histograms, properties)
        plot.draw_method = 'errorbar'
        compare_histograms(plot)
def compare_unfolding_methods(measurement='normalised_xsection',
                              add_before_unfolding=False,
                              channel='combined'):
    file_template = '/hdfs/TopQuarkGroup/run2/dpsData/'
    file_template += 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += '{measurement}_{channel}_RooUnfold{method}.txt'

    variables = [
        'MET', 'HT', 'ST', 'NJets', 'lepton_pt', 'abs_lepton_eta', 'WPT'
    ]
    for variable in variables:
        svd = file_template.format(variable=variable,
                                   method='Svd',
                                   channel=channel,
                                   measurement=measurement)
        bayes = file_template.format(variable=variable,
                                     method='Bayes',
                                     channel=channel,
                                     measurement=measurement)
        data = read_data_from_JSON(svd)
        before_unfolding = data['TTJet_measured_withoutFakes']
        svd_data = data['TTJet_unfolded']
        bayes_data = read_data_from_JSON(bayes)['TTJet_unfolded']
        h_svd = value_error_tuplelist_to_hist(svd_data,
                                              bin_edges_vis[variable])
        h_bayes = value_error_tuplelist_to_hist(bayes_data,
                                                bin_edges_vis[variable])
        h_before_unfolding = value_error_tuplelist_to_hist(
            before_unfolding, bin_edges_vis[variable])

        properties = Histogram_properties()
        properties.name = '{0}_compare_unfolding_methods_{1}_{2}'.format(
            measurement, variable, channel)
        properties.title = 'Comparison of unfolding methods'
        properties.path = 'plots'
        properties.has_ratio = True
        properties.xerr = True
        properties.x_limits = (bin_edges_vis[variable][0],
                               bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        if 'xsection' in measurement:
            properties.y_axis_title = r'$\frac{1}{\sigma}  \frac{d\sigma}{d' + \
                variables_latex[variable] + '}$'
        else:
            properties.y_axis_title = r'$t\bar{t}$ normalisation'

        histograms = {'SVD': h_svd, 'Bayes': h_bayes}
        if add_before_unfolding:
            histograms['before unfolding'] = h_before_unfolding
            properties.name += '_ext'
            properties.has_ratio = False
        plot = Plot(histograms, properties)
        plot.draw_method = 'errorbar'
        compare_histograms(plot)
def read_xsection_measurement_results(category, channel):
    global path_to_JSON, variable, k_value, met_type
    
    normalised_xsection_unfolded = None
    if category in met_uncertainties and variable == 'HT':
        normalised_xsection_unfolded = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + str(k_value) + '/' 
                                                       + 'central' + '/normalised_xsection_' + channel + '_' + met_type + '.txt')
    else:
        normalised_xsection_unfolded = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + str(k_value) + '/' 
                                                       + category + '/normalised_xsection_' + channel + '_' + met_type + '.txt')
        
    h_normalised_xsection = value_error_tuplelist_to_hist(normalised_xsection_unfolded['TTJet_measured'], bin_edges[variable])
    h_normalised_xsection_unfolded = value_error_tuplelist_to_hist(normalised_xsection_unfolded['TTJet_unfolded'], bin_edges[variable])
    
    
    histograms_normalised_xsection_different_generators = {'measured':h_normalised_xsection,
                                                           'unfolded':h_normalised_xsection_unfolded}
    
    histograms_normalised_xsection_systematics_shifts = {'measured':h_normalised_xsection,
                                                         'unfolded':h_normalised_xsection_unfolded}
    
    if category == 'central':
        # true distributions
        h_normalised_xsection_MADGRAPH = value_error_tuplelist_to_hist(normalised_xsection_unfolded['MADGRAPH'], bin_edges[variable])
        h_normalised_xsection_POWHEG = value_error_tuplelist_to_hist(normalised_xsection_unfolded['POWHEG'], bin_edges[variable])
        h_normalised_xsection_MCATNLO = value_error_tuplelist_to_hist(normalised_xsection_unfolded['MCATNLO'], bin_edges[variable])
        h_normalised_xsection_mathchingup = value_error_tuplelist_to_hist(normalised_xsection_unfolded['matchingup'], bin_edges[variable])
        h_normalised_xsection_mathchingdown = value_error_tuplelist_to_hist(normalised_xsection_unfolded['matchingdown'], bin_edges[variable])
        h_normalised_xsection_scaleup = value_error_tuplelist_to_hist(normalised_xsection_unfolded['scaleup'], bin_edges[variable])
        h_normalised_xsection_scaledown = value_error_tuplelist_to_hist(normalised_xsection_unfolded['scaledown'], bin_edges[variable])
        
        histograms_normalised_xsection_different_generators.update({'MADGRAPH':h_normalised_xsection_MADGRAPH,
                                                                    'POWHEG':h_normalised_xsection_POWHEG,
                                                                    'MCATNLO':h_normalised_xsection_MCATNLO})
        
        histograms_normalised_xsection_systematics_shifts.update({'MADGRAPH':h_normalised_xsection_MADGRAPH,
                                                                  'matchingdown': h_normalised_xsection_mathchingdown,
                                                                  'matchingup': h_normalised_xsection_mathchingup,
                                                                  'scaledown': h_normalised_xsection_scaledown,
                                                                  'scaleup': h_normalised_xsection_scaleup})
        
        normalised_xsection_unfolded_with_errors = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + 
                                                                   str(k_value) + '/' + category + '/normalised_xsection_' + 
                                                                   channel + '_' + met_type + '_with_errors.txt')
        # a rootpy.Graph with asymmetric errors!
        h_normalised_xsection_with_systematics = value_errors_tuplelist_to_graph(normalised_xsection_unfolded_with_errors['TTJet_measured'], bin_edges[variable])
        h_normalised_xsection_with_systematics_unfolded = value_errors_tuplelist_to_graph(normalised_xsection_unfolded_with_errors['TTJet_unfolded'], bin_edges[variable])
        
        histograms_normalised_xsection_different_generators['measured_with_systematics'] = h_normalised_xsection_with_systematics
        histograms_normalised_xsection_different_generators['unfolded_with_systematics'] = h_normalised_xsection_with_systematics_unfolded
        
        histograms_normalised_xsection_systematics_shifts['measured_with_systematics'] = h_normalised_xsection_with_systematics
        histograms_normalised_xsection_systematics_shifts['unfolded_with_systematics'] = h_normalised_xsection_with_systematics_unfolded
    
    return histograms_normalised_xsection_different_generators, histograms_normalised_xsection_systematics_shifts
def read_fit_templates_and_results_as_histograms(category, channel):
    global path_to_JSON, variable, met_type
    templates = read_data_from_JSON(
        path_to_JSON + "/" + variable + "/fit_results/" + category + "/templates_" + channel + "_" + met_type + ".txt"
    )
    data_values = read_data_from_JSON(
        path_to_JSON
        + "/"
        + variable
        + "/fit_results/"
        + category
        + "/initial_values_"
        + channel
        + "_"
        + met_type
        + ".txt"
    )["data"]
    fit_results = read_data_from_JSON(
        path_to_JSON + "/" + variable + "/fit_results/" + category + "/fit_results_" + channel + "_" + met_type + ".txt"
    )
    template_histograms = {}
    fit_results_histograms = {}
    for bin_i, variable_bin in enumerate(variable_bins_ROOT[variable]):
        h_template_data = value_tuplelist_to_hist(templates["data"][bin_i], eta_bin_edges)
        h_template_signal = value_tuplelist_to_hist(templates["signal"][bin_i], eta_bin_edges)
        h_template_VJets = value_tuplelist_to_hist(templates["V+Jets"][bin_i], eta_bin_edges)
        h_template_QCD = value_tuplelist_to_hist(templates["QCD"][bin_i], eta_bin_edges)
        template_histograms[variable_bin] = {
            "signal": h_template_signal,
            "V+Jets": h_template_VJets,
            "QCD": h_template_QCD,
        }
        h_data = h_template_data.Clone()
        h_signal = h_template_signal.Clone()
        h_VJets = h_template_VJets.Clone()
        h_QCD = h_template_QCD.Clone()

        data_normalisation = data_values[bin_i]
        signal_normalisation = fit_results["signal"][bin_i][0]
        VJets_normalisation = fit_results["V+Jets"][bin_i][0]
        QCD_normalisation = fit_results["QCD"][bin_i][0]

        h_data.Scale(data_normalisation)
        h_signal.Scale(signal_normalisation)
        h_VJets.Scale(VJets_normalisation)
        h_QCD.Scale(QCD_normalisation)
        h_background = h_VJets + h_QCD

        for bin_i in range(len(h_data)):
            h_data.SetBinError(bin_i + 1, sqrt(h_data.GetBinContent(bin_i + 1)))

        fit_results_histograms[variable_bin] = {"data": h_data, "signal": h_signal, "background": h_background}

    return template_histograms, fit_results_histograms
def main():
    global config, options
    parser = OptionParser()
    parser.add_option("-p",
                      "--path",
                      dest="path",
                      default='data/fit_checks/no_merging',
                      help="set path to JSON files")
    parser.add_option('--create_fit_data',
                      dest="create_fit_data",
                      action="store_true",
                      help="create the fit data for testing.")
    parser.add_option('--refit',
                      dest="refit",
                      action="store_true",
                      help="Fit again even if the output already exists")
    parser.add_option('--test',
                      dest="test",
                      action="store_true",
                      help="Test only: run just one selected sample")
    variables = config.histogram_path_templates.keys()
    fit_variables = fit_var_inputs
    mc_samples = ['TTJet', 'SingleTop', 'QCD', 'V+Jets']
    tests = closure_tests
    channels = ['electron', 'muon']
    COMEnergies = ['7', '8']

    (options, _) = parser.parse_args()
    print 'Running from path', options.path

    if (options.create_fit_data):
        create_fit_data(options.path,
                        variables,
                        fit_variables,
                        mc_samples,
                        COMEnergies=COMEnergies,
                        channels=channels)

    output_file = options.path + '/fit_test_output.txt'
    if options.test:
        output_file = options.path + '/fit_test_output_test.txt'
    if options.refit or not os.path.isfile(output_file) or options.test:
        if os.path.isfile(options.path + '/fit_check_data.txt'):
            fit_data = read_data_from_JSON(options.path +
                                           '/fit_check_data.txt')
            results = run_tests(fit_data, COMEnergies, variables,
                                fit_variables, mc_samples, channels, tests)
            write_data_to_JSON(results, output_file)
        else:
            print 'Please run bin/prepare_data_for_fit_checks first'
            print 'Then run this script with the option --create_fit_data.'

    results = read_data_from_JSON(output_file)
    plot_results(results)
def compare_combine_before_after_unfolding(measurement='normalised_xsection',
                              add_before_unfolding=False):
    file_template = 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += '{measurement}_{channel}_RooUnfold{method}.txt'

    variables = ['MET', 'HT', 'ST', 'NJets',
                 'lepton_pt', 'abs_lepton_eta', 'WPT']
    for variable in variables:
        combineBefore = file_template.format(
            variable=variable,
            method='Svd',
            channel='combinedBeforeUnfolding',
            measurement=measurement)
        combineAfter = file_template.format(
            variable=variable,
            method='Svd',
            channel='combined',
            measurement=measurement)
        data = read_data_from_JSON(combineBefore)
        before_unfolding = data['TTJet_measured']
        combineBefore_data = data['TTJet_unfolded']
        combineAfter_data = read_data_from_JSON(combineAfter)['TTJet_unfolded']
        h_combineBefore = value_error_tuplelist_to_hist(
            combineBefore_data, bin_edges_vis[variable])
        h_combineAfter = value_error_tuplelist_to_hist(
            combineAfter_data, bin_edges_vis[variable])
        h_before_unfolding = value_error_tuplelist_to_hist(
            before_unfolding, bin_edges_vis[variable])

        properties = Histogram_properties()
        properties.name = '{0}_compare_combine_before_after_unfolding_{1}'.format(
            measurement, variable)
        properties.title = 'Comparison of combining before/after unfolding'
        properties.path = 'plots'
        properties.has_ratio = True
        properties.xerr = True
        properties.x_limits = (
            bin_edges_vis[variable][0], bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        if 'xsection' in measurement:
            properties.y_axis_title = r'$\frac{1}{\sigma}  \frac{d\sigma}{d' + \
                variables_latex[variable] + '}$'
        else:
            properties.y_axis_title = r'$t\bar{t}$ normalisation'

        histograms = {'Combine before unfolding': h_combineBefore, 'Combine after unfolding': h_combineAfter}
        if add_before_unfolding:
            histograms['before unfolding'] = h_before_unfolding
            properties.name += '_ext'
            properties.has_ratio = False
        plot = Plot(histograms, properties)
        plot.draw_method = 'errorbar'
        compare_histograms(plot)
def compare_unfolding_methods(measurement='normalised_xsection',
                              add_before_unfolding=False, channel='combined'):
    file_template = '/hdfs/TopQuarkGroup/run2/dpsData/'
    file_template += 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += '{measurement}_{channel}_RooUnfold{method}.txt'

    variables = ['MET', 'HT', 'ST', 'NJets',
                 'lepton_pt', 'abs_lepton_eta', 'WPT']
    for variable in variables:
        svd = file_template.format(
            variable=variable,
            method='Svd',
            channel=channel,
            measurement=measurement)
        bayes = file_template.format(
            variable=variable,
            method='Bayes', channel=channel,
            measurement=measurement)
        data = read_data_from_JSON(svd)
        before_unfolding = data['TTJet_measured_withoutFakes']
        svd_data = data['TTJet_unfolded']
        bayes_data = read_data_from_JSON(bayes)['TTJet_unfolded']
        h_svd = value_error_tuplelist_to_hist(
            svd_data, bin_edges_vis[variable])
        h_bayes = value_error_tuplelist_to_hist(
            bayes_data, bin_edges_vis[variable])
        h_before_unfolding = value_error_tuplelist_to_hist(
            before_unfolding, bin_edges_vis[variable])

        properties = Histogram_properties()
        properties.name = '{0}_compare_unfolding_methods_{1}_{2}'.format(
            measurement, variable, channel)
        properties.title = 'Comparison of unfolding methods'
        properties.path = 'plots'
        properties.has_ratio = True
        properties.xerr = True
        properties.x_limits = (
            bin_edges_vis[variable][0], bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        if 'xsection' in measurement:
            properties.y_axis_title = r'$\frac{1}{\sigma}  \frac{d\sigma}{d' + \
                variables_latex[variable] + '}$'
        else:
            properties.y_axis_title = r'$t\bar{t}$ normalisation'

        histograms = {'SVD': h_svd, 'Bayes': h_bayes}
        if add_before_unfolding:
            histograms['before unfolding'] = h_before_unfolding
            properties.name += '_ext'
            properties.has_ratio = False
        plot = Plot(histograms, properties)
        plot.draw_method = 'errorbar'
        compare_histograms(plot)
Exemplo n.º 10
0
def read_xsection_measurement_results_with_errors(channel):
    global path_to_JSON, variable, k_values, met_type
    category = 'central'

    file_template = path_to_JSON + '/' + variable +  '/xsection_measurement_results/' + channel + '/kv' + str(k_values[channel]) + '/' + category + '/normalised_xsection_' + met_type + '.txt' 
    if channel == 'combined':
        file_template = file_template.replace('kv' + str(k_values[channel]), '')

    file_name = file_template
    normalised_xsection_unfolded = read_data_from_JSON( file_name )
    
    normalised_xsection_measured_unfolded = {'measured':normalised_xsection_unfolded['TTJet_measured'],
                                            'unfolded':normalised_xsection_unfolded['TTJet_unfolded']}
    
    file_name = file_template.replace('.txt', '_with_errors.txt')
    normalised_xsection_unfolded_with_errors = read_data_from_JSON( file_name )

    file_name = file_template.replace('.txt', '_ttbar_generator_errors.txt')
    normalised_xsection_ttbar_generator_errors = read_data_from_JSON( file_name )

    file_name = file_template.replace('.txt', '_MET_errors.txt')
    normalised_xsection_MET_errors = read_data_from_JSON( file_name )

    file_name = file_template.replace('.txt', '_topMass_errors.txt')
    normalised_xsection_topMass_errors = read_data_from_JSON( file_name )

    file_name = file_template.replace('.txt', '_kValue_errors.txt')
    normalised_xsection_kValue_errors = read_data_from_JSON( file_name )

    file_name = file_template.replace('.txt', '_PDF_errors.txt')
    normalised_xsection_PDF_errors = read_data_from_JSON( file_name )

    file_name = file_template.replace('.txt', '_other_errors.txt')
    normalised_xsection_other_errors = read_data_from_JSON( file_name )

    file_name = file_template.replace('.txt', '_new_errors.txt')
    normalised_xsection_new_errors = read_data_from_JSON( file_name )
    
    normalised_xsection_measured_unfolded.update({'measured_with_systematics':normalised_xsection_unfolded_with_errors['TTJet_measured'],
                                                'unfolded_with_systematics':normalised_xsection_unfolded_with_errors['TTJet_unfolded']})
    
    normalised_xsection_measured_errors = normalised_xsection_ttbar_generator_errors['TTJet_measured']
    normalised_xsection_measured_errors.update(normalised_xsection_PDF_errors['TTJet_measured'])
    normalised_xsection_measured_errors.update(normalised_xsection_MET_errors['TTJet_measured'])
    normalised_xsection_measured_errors.update(normalised_xsection_topMass_errors['TTJet_measured'])
    normalised_xsection_measured_errors.update(normalised_xsection_kValue_errors['TTJet_measured'])
    normalised_xsection_measured_errors.update(normalised_xsection_other_errors['TTJet_measured'])
    normalised_xsection_measured_errors.update(normalised_xsection_new_errors['TTJet_measured'])

    normalised_xsection_unfolded_errors = normalised_xsection_ttbar_generator_errors['TTJet_unfolded']
    normalised_xsection_unfolded_errors.update(normalised_xsection_PDF_errors['TTJet_unfolded'])
    normalised_xsection_unfolded_errors.update(normalised_xsection_MET_errors['TTJet_unfolded'])
    normalised_xsection_unfolded_errors.update(normalised_xsection_topMass_errors['TTJet_unfolded'])
    normalised_xsection_unfolded_errors.update(normalised_xsection_kValue_errors['TTJet_unfolded'])
    normalised_xsection_unfolded_errors.update(normalised_xsection_other_errors['TTJet_unfolded'])
    normalised_xsection_unfolded_errors.update(normalised_xsection_new_errors['TTJet_unfolded'])

    return normalised_xsection_measured_unfolded, normalised_xsection_measured_errors, normalised_xsection_unfolded_errors
def compare_combine_before_after_unfolding_uncertainties():
    file_template = 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += 'unfolded_normalisation_{channel}_RooUnfoldSvd.txt'

    variables = [
        'MET', 'HT', 'ST', 'NJets', 'lepton_pt', 'abs_lepton_eta', 'WPT'
    ]
    #     variables = ['ST']
    for variable in variables:
        beforeUnfolding = file_template.format(
            variable=variable, channel='combinedBeforeUnfolding')
        afterUnfolding = file_template.format(variable=variable,
                                              channel='combined')
        data = read_data_from_JSON(beforeUnfolding)
        before_unfolding = data['TTJet_measured']
        beforeUnfolding_data = data['TTJet_unfolded']
        afterUnfolding_data = read_data_from_JSON(
            afterUnfolding)['TTJet_unfolded']

        before_unfolding = [e / v * 100 for v, e in before_unfolding]
        beforeUnfolding_data = [e / v * 100 for v, e in beforeUnfolding_data]
        afterUnfolding_data = [e / v * 100 for v, e in afterUnfolding_data]

        h_beforeUnfolding = value_tuplelist_to_hist(beforeUnfolding_data,
                                                    bin_edges_vis[variable])
        h_afterUnfolding = value_tuplelist_to_hist(afterUnfolding_data,
                                                   bin_edges_vis[variable])
        h_before_unfolding = value_tuplelist_to_hist(before_unfolding,
                                                     bin_edges_vis[variable])

        properties = Histogram_properties()
        properties.name = 'compare_combine_before_after_unfolding_uncertainties_{0}'.format(
            variable)
        properties.title = 'Comparison of unfolding uncertainties'
        properties.path = 'plots'
        properties.has_ratio = False
        properties.xerr = True
        properties.x_limits = (bin_edges_vis[variable][0],
                               bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        properties.y_axis_title = 'relative uncertainty (\\%)'
        properties.legend_location = (0.98, 0.95)

        histograms = {
            'Combine before unfolding': h_beforeUnfolding,
            'Combine after unfolding': h_afterUnfolding,
            # 'before unfolding': h_before_unfolding
        }
        plot = Plot(histograms, properties)
        plot.draw_method = 'errorbar'
        compare_histograms(plot)
def read_normalised_xsection_measurement(category, channel):
    global path_to_JSON, met_type, met_uncertainties
    normalised_xsection = None
    
    if category in met_uncertainties and variable == 'HT':
        normalised_xsection = read_data_from_JSON(path_to_JSON + 'central' + '/normalised_xsection_' + channel + '_' + met_type + '.txt')
    else:
        normalised_xsection = read_data_from_JSON(path_to_JSON + category + '/normalised_xsection_' + channel + '_' + met_type + '.txt')
    
    measurement = normalised_xsection['TTJet_measured']
    measurement_unfolded = normalised_xsection['TTJet_unfolded']
    
    return measurement, measurement_unfolded
def compare_combine_before_after_unfolding_uncertainties():
    file_template = 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += 'unfolded_normalisation_{channel}_RooUnfoldSvd.txt'

    variables = ['MET', 'HT', 'ST', 'NJets',
                 'lepton_pt', 'abs_lepton_eta', 'WPT']
#     variables = ['ST']
    for variable in variables:
        beforeUnfolding = file_template.format(
            variable=variable, channel='combinedBeforeUnfolding')
        afterUnfolding = file_template.format(
            variable=variable, channel='combined')
        data = read_data_from_JSON(beforeUnfolding)
        before_unfolding = data['TTJet_measured']
        beforeUnfolding_data = data['TTJet_unfolded']
        afterUnfolding_data = read_data_from_JSON(afterUnfolding)['TTJet_unfolded']

        before_unfolding = [e / v * 100 for v, e in before_unfolding]
        beforeUnfolding_data = [e / v * 100 for v, e in beforeUnfolding_data]
        afterUnfolding_data = [e / v * 100 for v, e in afterUnfolding_data]

        h_beforeUnfolding = value_tuplelist_to_hist(
            beforeUnfolding_data, bin_edges_vis[variable])
        h_afterUnfolding = value_tuplelist_to_hist(
            afterUnfolding_data, bin_edges_vis[variable])
        h_before_unfolding = value_tuplelist_to_hist(
            before_unfolding, bin_edges_vis[variable])

        properties = Histogram_properties()
        properties.name = 'compare_combine_before_after_unfolding_uncertainties_{0}'.format(
            variable)
        properties.title = 'Comparison of unfolding uncertainties'
        properties.path = 'plots'
        properties.has_ratio = False
        properties.xerr = True
        properties.x_limits = (
            bin_edges_vis[variable][0], bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        properties.y_axis_title = 'relative uncertainty (\\%)'
        properties.legend_location = (0.98, 0.95)

        histograms = {'Combine before unfolding': h_beforeUnfolding, 'Combine after unfolding': h_afterUnfolding,
                      # 'before unfolding': h_before_unfolding
                      }
        plot = Plot(histograms, properties)
        plot.draw_method = 'errorbar'
        compare_histograms(plot)
def compare_unfolding_uncertainties():
    file_template = '/hdfs/TopQuarkGroup/run2/dpsData/'
    file_template += 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += 'unfolded_normalisation_combined_RooUnfold{method}.txt'

    variables = ['MET', 'HT', 'ST', 'NJets',
                 'lepton_pt', 'abs_lepton_eta', 'WPT']
#     variables = ['ST']
    for variable in variables:
        svd = file_template.format(
            variable=variable, method='Svd')
        bayes = file_template.format(
            variable=variable, method='Bayes')
        data = read_data_from_JSON(svd)
        before_unfolding = data['TTJet_measured_withoutFakes']
        svd_data = data['TTJet_unfolded']
        bayes_data = read_data_from_JSON(bayes)['TTJet_unfolded']

        before_unfolding = [e / v * 100 for v, e in before_unfolding]
        svd_data = [e / v * 100 for v, e in svd_data]
        bayes_data = [e / v * 100 for v, e in bayes_data]

        h_svd = value_tuplelist_to_hist(
            svd_data, bin_edges_vis[variable])
        h_bayes = value_tuplelist_to_hist(
            bayes_data, bin_edges_vis[variable])
        h_before_unfolding = value_tuplelist_to_hist(
            before_unfolding, bin_edges_vis[variable])

        properties = Histogram_properties()
        properties.name = 'compare_unfolding_uncertainties_{0}'.format(
            variable)
        properties.title = 'Comparison of unfolding uncertainties'
        properties.path = 'plots'
        properties.has_ratio = False
        properties.xerr = True
        properties.x_limits = (
            bin_edges_vis[variable][0], bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        properties.y_axis_title = 'relative uncertainty (\\%)'
        properties.legend_location = (0.98, 0.95)

        histograms = {'SVD': h_svd, 'Bayes': h_bayes,
                      'before unfolding': h_before_unfolding}
        plot = Plot(histograms, properties)
        plot.draw_method = 'errorbar'
        compare_histograms(plot)
def compare_unfolding_uncertainties():
    file_template = '/hdfs/TopQuarkGroup/run2/dpsData/'
    file_template += 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += 'unfolded_normalisation_combined_RooUnfold{method}.txt'

    variables = [
        'MET', 'HT', 'ST', 'NJets', 'lepton_pt', 'abs_lepton_eta', 'WPT'
    ]
    #     variables = ['ST']
    for variable in variables:
        svd = file_template.format(variable=variable, method='Svd')
        bayes = file_template.format(variable=variable, method='Bayes')
        data = read_data_from_JSON(svd)
        before_unfolding = data['TTJet_measured_withoutFakes']
        svd_data = data['TTJet_unfolded']
        bayes_data = read_data_from_JSON(bayes)['TTJet_unfolded']

        before_unfolding = [e / v * 100 for v, e in before_unfolding]
        svd_data = [e / v * 100 for v, e in svd_data]
        bayes_data = [e / v * 100 for v, e in bayes_data]

        h_svd = value_tuplelist_to_hist(svd_data, bin_edges_vis[variable])
        h_bayes = value_tuplelist_to_hist(bayes_data, bin_edges_vis[variable])
        h_before_unfolding = value_tuplelist_to_hist(before_unfolding,
                                                     bin_edges_vis[variable])

        properties = Histogram_properties()
        properties.name = 'compare_unfolding_uncertainties_{0}'.format(
            variable)
        properties.title = 'Comparison of unfolding uncertainties'
        properties.path = 'plots'
        properties.has_ratio = False
        properties.xerr = True
        properties.x_limits = (bin_edges_vis[variable][0],
                               bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        properties.y_axis_title = 'relative uncertainty (\\%)'
        properties.legend_location = (0.98, 0.95)

        histograms = {
            'SVD': h_svd,
            'Bayes': h_bayes,
            'before unfolding': h_before_unfolding
        }
        plot = Plot(histograms, properties)
        plot.draw_method = 'errorbar'
        compare_histograms(plot)
Exemplo n.º 16
0
def read_normalised_xsection_measurement(options, category):
    '''
    Returns the normalised measurement and normalised unfolded measurement for 
    the file associated with the variable under study
    '''
    variable=options['variable']
    variables_no_met=options['variables_no_met']
    met_specific_systematics=options['met_specific_systematics']
    path_to_JSON=options['path_to_JSON']
    method=options['method']
    channel=options['channel']
    filename = '{path}/{category}/normalised_xsection_{channel}_{method}.txt'
    # Disregarding Met Uncertainties if variable does not use MET
    if (category in met_specific_systematics) and (variable in variables_no_met):
        filename = filename.format(
            path = path_to_JSON,
            channel = channel,
            category = 'central',
            method = method,
        )
    else:
        filename = filename.format(
            path = path_to_JSON,
            channel = channel,
            category = category,
            method = method
        )

    normalised_xsection = read_data_from_JSON( filename )
    measurement = normalised_xsection['TTJet_measured']#should this be measured without fakes???
    measurement_unfolded = normalised_xsection['TTJet_unfolded']
    return measurement, measurement_unfolded  
def parse_options():
    parser = OptionParser(__doc__)
    parser.add_option("-c",
                      "--compare",
                      dest="compare",
                      action="store_true",
                      help="Compare to current values (k vs tau)",
                      default=False)
    parser.add_option(
        "-t",
        "--table-style",
        dest="style",
        default='simple',
        help="Style for table printing: simple|latex|twiki (default = simple)")

    (options, args) = parser.parse_args()

    input_values_sets = []
    json_input_files = []
    add_set = input_values_sets.append
    add_json_file = json_input_files.append
    for arg in args:
        input_values = read_data_from_JSON(arg)
        add_set(input_values)
        add_json_file(arg)

    return options, input_values_sets, json_input_files
Exemplo n.º 18
0
def read_unfolded_xsections(channel):
    global path_to_JSON, variable, k_value, met_type, b_tag_bin
    TTJet_xsection_unfolded = {}
    for category in categories:
        normalised_xsections = read_data_from_JSON(path_to_JSON  + '/xsection_measurement_results' + '/kv' + str(k_value) + '/' + category + '/normalised_xsection_' + channel + '_' + met_type + '.txt')
        TTJet_xsection_unfolded[category] = normalised_xsections['TTJet_unfolded']
    return TTJet_xsection_unfolded
def get_fitted_normalisation(variable, channel):
    global path_to_JSON, category, met_type
    fit_results = read_data_from_JSON(path_to_JSON + variable +
                                      '/fit_results/' + category +
                                      '/fit_results_' + channel + '_' +
                                      met_type + '.txt')

    N_fit_ttjet = [0, 0]
    N_fit_singletop = [0, 0]
    N_fit_vjets = [0, 0]
    N_fit_qcd = [0, 0]

    bins = variable_bins_ROOT[variable]
    for bin_i, _ in enumerate(bins):
        #central values
        N_fit_ttjet[0] += fit_results['TTJet'][bin_i][0]
        N_fit_singletop[0] += fit_results['SingleTop'][bin_i][0]
        N_fit_vjets[0] += fit_results['V+Jets'][bin_i][0]
        N_fit_qcd[0] += fit_results['QCD'][bin_i][0]

        #errors
        N_fit_ttjet[1] += fit_results['TTJet'][bin_i][1]
        N_fit_singletop[1] += fit_results['SingleTop'][bin_i][1]
        N_fit_vjets[1] += fit_results['V+Jets'][bin_i][1]
        N_fit_qcd[1] += fit_results['QCD'][bin_i][1]

    fitted_normalisation = {
        'TTJet': N_fit_ttjet,
        'SingleTop': N_fit_singletop,
        'V+Jets': N_fit_vjets,
        'QCD': N_fit_qcd
    }
    return fitted_normalisation
def get_fitted_normalisation(variable, channel):
    global path_to_JSON, category, met_type
    fit_results = read_data_from_JSON(path_to_JSON + variable + '/fit_results/' + category + '/fit_results_' + channel + '_' + met_type + '.txt')

    N_fit_ttjet = [0, 0]
    N_fit_singletop = [0, 0]
    N_fit_vjets = [0, 0]
    N_fit_qcd = [0, 0]

    bins = variable_bins_ROOT[variable]
    for bin_i, _ in enumerate(bins):
        #central values
        N_fit_ttjet[0] += fit_results['TTJet'][bin_i][0]
        N_fit_singletop[0] += fit_results['SingleTop'][bin_i][0]
        N_fit_vjets[0] += fit_results['V+Jets'][bin_i][0]
        N_fit_qcd[0] += fit_results['QCD'][bin_i][0]

        #errors
        N_fit_ttjet[1] += fit_results['TTJet'][bin_i][1]
        N_fit_singletop[1] += fit_results['SingleTop'][bin_i][1]
        N_fit_vjets[1] += fit_results['V+Jets'][bin_i][1]
        N_fit_qcd[1] += fit_results['QCD'][bin_i][1]

    fitted_normalisation = {
                'TTJet': N_fit_ttjet,
                'SingleTop': N_fit_singletop,
                'V+Jets': N_fit_vjets,
                'QCD': N_fit_qcd
                }
    return fitted_normalisation
def get_fit_results(variable, channel):
    global path_to_JSON, category, met_type
    fit_results = read_data_from_JSON(path_to_JSON + variable +
                                      '/fit_results/' + category +
                                      '/fit_results_' + channel + '_' +
                                      met_type + '.txt')
    return fit_results
Exemplo n.º 22
0
def get_fitted_normalisation( variable, channel, path_to_JSON, category, met_type ):
    '''
    This function now gets the error on the fit correctly,
    so that it can be applied if the --normalise_to_fit option is used
    '''
    import config.variable_binning
    variable_bins_ROOT = config.variable_binning.variable_bins_ROOT 
    fit_results = read_data_from_JSON( path_to_JSON + variable + '/fit_results/' + category + '/fit_results_' + channel + '_' + met_type + '.txt' )

    N_fit_ttjet = [0, 0]
    N_fit_singletop = [0, 0]
    N_fit_vjets = [0, 0]
    N_fit_qcd = [0, 0]

    bins = variable_bins_ROOT[variable]
    for bin_i, _ in enumerate( bins ):
        # central values
        N_fit_ttjet[0] += fit_results['TTJet'][bin_i][0]
        N_fit_singletop[0] += fit_results['SingleTop'][bin_i][0]
        N_fit_vjets[0] += fit_results['V+Jets'][bin_i][0]
        N_fit_qcd[0] += fit_results['QCD'][bin_i][0]

        # errors
        N_fit_ttjet[1] += fit_results['TTJet'][bin_i][1]
        N_fit_singletop[1] += fit_results['SingleTop'][bin_i][1]
        N_fit_vjets[1] += fit_results['V+Jets'][bin_i][1]
        N_fit_qcd[1] += fit_results['QCD'][bin_i][1]

    fitted_normalisation = {
                'TTJet': N_fit_ttjet,
                'SingleTop': N_fit_singletop,
                'V+Jets': N_fit_vjets,
                'QCD': N_fit_qcd
                }
    return fitted_normalisation
Exemplo n.º 23
0
def main(options, args):
    config = XSectionConfig(options.CoM)
    variables = ['MET', 'HT', 'ST', 'WPT']
    channels = ['electron', 'muon', 'combined']
    m_file = 'normalised_xsection_patType1CorrectedPFMet.txt'
    m_with_errors_file = 'normalised_xsection_patType1CorrectedPFMet_with_errors.txt'
    path_template = args[0]
    output_file = 'measurement_{0}TeV.root'.format(options.CoM)
    f = File(output_file, 'recreate')
    for channel in channels:
        d = f.mkdir(channel)
        d.cd()
        for variable in variables:
            dv = d.mkdir(variable)
            dv.cd()
            if channel == 'combined':
                path = path_template.format(variable=variable,
                                            channel=channel,
                                            centre_of_mass_energy=options.CoM)
            else:
                kv = channel + \
                    '/kv{0}/'.format(config.k_values[channel][variable])
                path = path_template.format(variable=variable,
                                            channel=kv,
                                            centre_of_mass_energy=options.CoM)

            m = read_data_from_JSON(path + '/' + m_file)
            m_with_errors = read_data_from_JSON(path + '/' +
                                                m_with_errors_file)

            for name, result in m.items():
                h = make_histogram(result, bin_edges[variable])
                h.SetName(name)
                h.write()

            for name, result in m_with_errors.items():
                if not 'TTJet' in name:
                    continue
                h = make_histogram(result, bin_edges[variable])
                h.SetName(name + '_with_syst')
                h.write()
            dv.write()
            d.cd()
        d.write()
    f.write()
    f.close()
def get_data_histogram( channel, variable, met_type ):
    fit_result_input = 'data/M3_angle_bl/13TeV/%(variable)s/fit_results/central/fit_results_%(channel)s_%(met_type)s.txt'
    fit_results = read_data_from_JSON( fit_result_input % {'channel': channel, 'variable': variable, 'met_type':met_type} )
    fit_data = fit_results['TTJet']
    print fit_data
    print bin_edges[variable]
    h_data = value_error_tuplelist_to_hist( fit_data, bin_edges[variable] )
    return h_data
Exemplo n.º 25
0
def main(options, args):
    config = XSectionConfig(options.CoM)
    variables = ['MET', 'HT', 'ST', 'WPT']
    channels = ['electron', 'muon', 'combined']
    m_file = 'normalised_xsection_patType1CorrectedPFMet.txt'
    m_with_errors_file = 'normalised_xsection_patType1CorrectedPFMet_with_errors.txt'
    path_template = args[0]
    output_file = 'measurement_{0}TeV.root'.format(options.CoM)
    f = File(output_file, 'recreate')
    for channel in channels:
        d = f.mkdir(channel)
        d.cd()
        for variable in variables:
            dv = d.mkdir(variable)
            dv.cd()
            if channel == 'combined':
                path = path_template.format(variable=variable,
                                            channel=channel,
                                            centre_of_mass_energy=options.CoM)
            else:
                kv = channel + \
                    '/kv{0}/'.format(config.k_values[channel][variable])
                path = path_template.format(variable=variable,
                                            channel=kv,
                                            centre_of_mass_energy=options.CoM)

            m = read_data_from_JSON(path + '/' + m_file)
            m_with_errors = read_data_from_JSON(
                path + '/' + m_with_errors_file)

            for name, result in m.items():
                h = make_histogram(result, bin_edges[variable])
                h.SetName(name)
                h.write()

            for name, result in m_with_errors.items():
                if not 'TTJet' in name:
                    continue
                h = make_histogram(result, bin_edges[variable])
                h.SetName(name + '_with_syst')
                h.write()
            dv.write()
            d.cd()
        d.write()
    f.write()
    f.close()
def main():
    global config, options
    parser = OptionParser()
    parser.add_option( "-p", "--path", dest = "path", default = 'data/fit_checks/no_merging',
                  help = "set path to JSON files" )
    parser.add_option( '--create_fit_data', dest = "create_fit_data", action = "store_true",
                      help = "create the fit data for testing." )
    parser.add_option( '--refit', dest = "refit", action = "store_true",
                      help = "Fit again even if the output already exists" )
    parser.add_option( '--test', dest = "test", action = "store_true",
                      help = "Test only: run just one selected sample" )
    variables = config.histogram_path_templates.keys()
    fit_variables = fit_var_inputs
    mc_samples = ['TTJet', 'SingleTop', 'QCD', 'V+Jets']
    tests = closure_tests
    channels = ['electron', 'muon']
    COMEnergies = ['7', '8']
    
    ( options, _ ) = parser.parse_args()
    print 'Running from path', options.path 
    
    if ( options.create_fit_data ):
        create_fit_data( options.path, variables, fit_variables, mc_samples,
                        COMEnergies = COMEnergies, channels = channels )
     
    output_file = options.path + '/fit_test_output.txt'
    if options.test:
        output_file = options.path + '/fit_test_output_test.txt'
    if options.refit or not os.path.isfile( output_file ) or options.test:
        if os.path.isfile( options.path + '/fit_check_data.txt' ):
            fit_data = read_data_from_JSON( options.path + '/fit_check_data.txt' )
            results = run_tests( fit_data,
                                COMEnergies,
                                variables,
                                fit_variables,
                                mc_samples,
                                channels,
                                tests )
            write_data_to_JSON(results, output_file )
        else:
            print 'Please run bin/prepare_data_for_fit_checks first'
            print 'Then run this script with the option --create_fit_data.'
     
    results = read_data_from_JSON(output_file)
    plot_results( results )
Exemplo n.º 27
0
def read_xsection_measurement_results(category, channel):
    global path_to_JSON, variable, k_value, met_type
    
    normalised_xsection_unfolded = read_data_from_JSON(path_to_JSON  + '/xsection_measurement_results' + '/kv' + str(k_value) + '/' 
                                                       + category + '/normalised_xsection_' + channel + '_' + met_type + '.txt')
    h_normalised_xsection = value_error_tuplelist_to_hist(normalised_xsection_unfolded['TTJet_measured'], bin_edges[variable])
    h_normalised_xsection_unfolded = value_error_tuplelist_to_hist(normalised_xsection_unfolded['TTJet_unfolded'], bin_edges[variable])
    
    if category == 'central':    
        normalised_xsection_unfolded_with_errors = read_data_from_JSON(path_to_JSON  + '/xsection_measurement_results' + '/kv' + 
                                                                   str(k_value) + '/' + category + '/normalised_xsection_' + 
                                                                   channel + '_' + met_type + '_with_errors.txt')
        h_normalised_xsection = value_errors_tuplelist_to_graph(normalised_xsection_unfolded_with_errors['TTJet_measured'], bin_edges[variable])
        h_normalised_xsection_unfolded = value_errors_tuplelist_to_graph(normalised_xsection_unfolded_with_errors['TTJet_unfolded'], bin_edges[variable])
    
    
    #true distributions
    h_normalised_xsection_MADGRAPH = value_error_tuplelist_to_hist(normalised_xsection_unfolded['MADGRAPH'], bin_edges[variable])
    h_normalised_xsection_POWHEG = value_error_tuplelist_to_hist(normalised_xsection_unfolded['POWHEG'], bin_edges[variable])
    h_normalised_xsection_MCATNLO = value_error_tuplelist_to_hist(normalised_xsection_unfolded['MCATNLO'], bin_edges[variable])
    h_normalised_xsection_mathchingup = value_error_tuplelist_to_hist(normalised_xsection_unfolded['matchingup'], bin_edges[variable])
    h_normalised_xsection_mathchingdown = value_error_tuplelist_to_hist(normalised_xsection_unfolded['matchingdown'], bin_edges[variable])
    h_normalised_xsection_scaleup = value_error_tuplelist_to_hist(normalised_xsection_unfolded['scaleup'], bin_edges[variable])
    h_normalised_xsection_scaledown = value_error_tuplelist_to_hist(normalised_xsection_unfolded['scaledown'], bin_edges[variable])
    
    histograms_normalised_xsection_different_generators = {
                  'measured':h_normalised_xsection,
                  'unfolded':h_normalised_xsection_unfolded,
                  'MADGRAPH':h_normalised_xsection_MADGRAPH,
                  'POWHEG':h_normalised_xsection_POWHEG,
                  'MCATNLO':h_normalised_xsection_MCATNLO
                  }
    
    histograms_normalised_xsection_systematics_shifts = {
                  'measured':h_normalised_xsection,
                  'unfolded':h_normalised_xsection_unfolded,
                  'matchingdown': h_normalised_xsection_mathchingdown,
                  'matchingup': h_normalised_xsection_mathchingup,
                  'scaledown': h_normalised_xsection_scaledown,
                  'scaleup': h_normalised_xsection_scaleup
                  }
    
    return histograms_normalised_xsection_different_generators, histograms_normalised_xsection_systematics_shifts
Exemplo n.º 28
0
def read_fit_templates_and_results_as_histograms(category, channel):
    global path_to_JSON, variable, met_type
    templates = read_data_from_JSON(path_to_JSON  + '/fit_results/' + category + '/templates_' + channel + '_' + met_type + '.txt')
    data_values = read_data_from_JSON(path_to_JSON  + '/fit_results/' + category + '/initial_values_' + channel + '_' + met_type + '.txt')['data']
    fit_results = read_data_from_JSON(path_to_JSON  + '/fit_results/' + category + '/fit_results_' + channel + '_' + met_type + '.txt')
    template_histograms = {}
    fit_results_histograms = {}
    for bin_i, variable_bin in enumerate(variable_bins_ROOT[variable]):
        h_template_data = value_tuplelist_to_hist(templates['data'][bin_i], eta_bin_edges)
        h_template_signal = value_tuplelist_to_hist(templates['signal'][bin_i], eta_bin_edges)
        h_template_VJets = value_tuplelist_to_hist(templates['V+Jets'][bin_i], eta_bin_edges)
        h_template_QCD = value_tuplelist_to_hist(templates['QCD'][bin_i], eta_bin_edges)
        template_histograms[variable_bin] = {
                                    'signal':h_template_signal,
                                    'V+Jets':h_template_VJets,
                                    'QCD':h_template_QCD
                                    }
        h_data = h_template_data.Clone()
        h_signal = h_template_signal.Clone()
        h_VJets = h_template_VJets.Clone()
        h_QCD = h_template_QCD.Clone()
        
        data_normalisation = data_values[bin_i]
        signal_normalisation = fit_results['signal'][bin_i][0]
        VJets_normalisation = fit_results['V+Jets'][bin_i][0]
        QCD_normalisation = fit_results['QCD'][bin_i][0]
        
        h_data.Scale(data_normalisation)
        h_signal.Scale(signal_normalisation)
        h_VJets.Scale(VJets_normalisation)
        h_QCD.Scale(QCD_normalisation)
        h_background = h_VJets + h_QCD
        
        for bin_i in range(len(h_data)):
            h_data.SetBinError(bin_i+1, sqrt(h_data.GetBinContent(bin_i+1)))
        
        fit_results_histograms[variable_bin] = {
                                                'data':h_data,
                                                'signal':h_signal,
                                                'background':h_background
                                                }
        
    return template_histograms, fit_results_histograms
Exemplo n.º 29
0
def get_fit_results_histogram( data_path = 'data/M3_angle_bl',
                               centre_of_mass = 8,
                               channel = 'electron',
                               variable = 'MET',
                               met_type = 'patType1CorrectedPFMet',
                               bin_edges = [] ):
    fit_result_input = data_path + '/%(CoM)dTeV/%(variable)s/fit_results/central/fit_results_%(channel)s_%(met_type)s.txt'
    fit_results = read_data_from_JSON( fit_result_input % {'CoM': centre_of_mass, 'channel': channel, 'variable': variable, 'met_type':met_type} )
    fit_data = fit_results['TTJet']
    h_data = value_error_tuplelist_to_hist( fit_data, bin_edges )
    return h_data
Exemplo n.º 30
0
def read_from_fit_results_folder(path_to_JSON = 'data', 
                     variable = 'MET', 
                     category = 'central',
                     channel = 'combined', 
                     met_type = 'patType1CorrectedPFMet',
                     data_type = 'fit_results'):
    filename = path_to_JSON + '/' + variable + '/fit_results/' + category + '/'
    filename += data_type + '_' + channel + '_' + met_type + '.txt'
    results = read_data_from_JSON( filename )
    
    return results
Exemplo n.º 31
0
def read_from_fit_results_folder(path_to_JSON='data',
                                 variable='MET',
                                 category='central',
                                 channel='combined',
                                 met_type='patType1CorrectedPFMet',
                                 data_type='fit_results'):
    filename = path_to_JSON + '/' + category + '/'
    filename += data_type + '_' + channel + '_' + met_type + '.txt'
    results = read_data_from_JSON(filename)

    return results
Exemplo n.º 32
0
def read_unfolded_xsections(channel):
    global path_to_JSON, variable, k_value, met_type, b_tag_bin
    TTJet_xsection_unfolded = {}
    for category in categories:
        normalised_xsections = read_data_from_JSON(
            path_to_JSON + '/' + variable + '/xsection_measurement_results' +
            '/kv' + str(k_value) + '/' + category + '/normalised_xsection_' +
            channel + '_' + met_type + '.txt')
        TTJet_xsection_unfolded[category] = normalised_xsections[
            'TTJet_unfolded']
    return TTJet_xsection_unfolded
Exemplo n.º 33
0
def get_data(file_name, subset=""):
    # this takes a LOT of memory, please use subset!!
    all_data = []
    extend = all_data.extend
    data = read_data_from_JSON(file_name)

    if subset:
        for entry in data:  # loop over all data entries
            extend(entry[subset])
    else:
        extend(data)

    return all_data
def get_data(files, subset=''):
    # this takes a LOT of memory, please use subset!!
    all_data = []
    extend = all_data.extend
    for f in files:
        data = read_data_from_JSON(f)

        if subset:
            for entry in data:  # loop over all data entries
                extend(entry[subset])
        else:
            extend(data)

    return all_data
Exemplo n.º 35
0
def read_unfolded_normalisation(path_to_JSON='data',
                                variable='MET',
                                category='central',
                                channel='combined',
                                met_type='patType1CorrectedPFMet'):
    new_path = '{path}/xsection_measurement_results/{channel}/{category}/{file}'
    result_file = 'normalisation_{0}.txt'.format(met_type)
    new_path = new_path.format(
        path=path_to_JSON,
        channel=channel,
        category=category,
        file=result_file,
    )
    return read_data_from_JSON(new_path)
Exemplo n.º 36
0
def read_unfolded_normalisation(
        path_to_JSON='data',
        variable='MET',
        category='central',
        channel='combined',
        met_type='patType1CorrectedPFMet'):
    new_path = '{path}/xsection_measurement_results/{channel}/{category}/{file}'
    result_file = 'normalisation_{0}.txt'.format(met_type)
    new_path = new_path.format(
        path=path_to_JSON,
        channel=channel,
        category=category,
        file=result_file,
    )
    return read_data_from_JSON(new_path)
Exemplo n.º 37
0
def parse_options():
    parser = OptionParser( __doc__ )

    ( options, args ) = parser.parse_args()
    
    input_values_sets = []
    json_input_files = []
    add_set = input_values_sets.append
    add_json_file = json_input_files.append
    for arg in args:
        input_values = read_data_from_JSON( arg )
        add_set( input_values )
        add_json_file( arg )

    return options, input_values_sets, json_input_files
Exemplo n.º 38
0
    def __set_unfolding_histograms__( self ):
        # at the moment only one file is supported for the unfolding input
        files = set( [self.truth['file'],
                     self.gen_vs_reco['file'],
                     self.measured['file']]
                    )
        if len( files ) > 1:
            print "Currently not supported to have different files for truth, gen_vs_reco and measured"
            sys.exit()
            
        input_file = files.pop()

        visiblePS = False
        if self.phaseSpace == 'VisiblePS':
            visiblePS = True

        t, m, r, f = get_unfold_histogram_tuple( File(input_file),
                                              self.variable,
                                              self.channel,
                                              centre_of_mass = self.centre_of_mass_energy,
                                              ttbar_xsection=self.measurement_config.ttbar_xsection,
                                              luminosity=self.measurement_config.luminosity,
                                              load_fakes = True,
                                              visiblePS = visiblePS
                                            )
        self.h_truth = asrootpy ( t )
        self.h_response = asrootpy ( r )
        self.h_measured = asrootpy ( m )
        self.h_fakes = asrootpy ( f )
        
        data_file = self.data['file']
        if data_file.endswith('.root'):
            self.h_data = get_histogram_from_file(self.data['histogram'], self.data['file'])
        elif data_file.endswith('.json') or data_file.endswith('.txt'):
            data_key = self.data['histogram']
            # assume configured bin edges
            edges = []
            edges = reco_bin_edges_vis[self.variable]

            json_input = read_data_from_JSON(data_file)

            if data_key == "": # JSON file == histogram
                self.h_data = value_error_tuplelist_to_hist(json_input, edges)
            else:
                self.h_data = value_error_tuplelist_to_hist(json_input[data_key], edges)
        else:
            print 'Unkown file extension', data_file.split('.')[-1]
Exemplo n.º 39
0
def get_fit_results_histogram(data_path='data/M3_angle_bl',
                              centre_of_mass=8,
                              channel='electron',
                              variable='MET',
                              met_type='patType1CorrectedPFMet',
                              bin_edges=[]):
    fit_result_input = data_path + '/%(CoM)dTeV/%(variable)s/fit_results/central/fit_results_%(channel)s_%(met_type)s.txt'
    fit_results = read_data_from_JSON(
        fit_result_input % {
            'CoM': centre_of_mass,
            'channel': channel,
            'variable': variable,
            'met_type': met_type
        })
    fit_data = fit_results['TTJet']
    h_data = value_error_tuplelist_to_hist(fit_data, bin_edges)
    return h_data
def read_normalised_xsection_measurement( category, channel ):
    global path_to_JSON, met_type, met_uncertainties_list, k_values
    filename = ''
    
    if category in met_uncertainties_list and variable == 'HT':
        filename = path_to_JSON + '/' + channel + '/kv' + str( k_values[channel] ) + '/central/normalised_xsection_' + met_type + '.txt' 
    else:
        filename = path_to_JSON + '/' + channel + '/kv' + str( k_values[channel] ) + '/' + category + '/normalised_xsection_' + met_type + '.txt' 
    
    if channel == 'combined':
        filename = filename.replace( 'kv' + str( k_values[channel] ), '' )

    normalised_xsection = read_data_from_JSON( filename )
    
    measurement = normalised_xsection['TTJet_measured']
    measurement_unfolded = normalised_xsection['TTJet_unfolded']
    
    return measurement, measurement_unfolded
def parse_options():
    parser = OptionParser( __doc__ )
    parser.add_option( "-c", "--compare", dest = "compare", action = "store_true",
                      help = "Compare to current values (k vs tau)", default = False )
    parser.add_option( "-t", "--table-style", dest = "style", default = 'simple',
                      help = "Style for table printing: simple|latex|twiki (default = simple)" )
    ( options, args ) = parser.parse_args()
    
    input_values_sets = []
    json_input_files = []
    add_set = input_values_sets.append
    add_json_file = json_input_files.append
    for arg in args:
        input_values = read_data_from_JSON( arg )
        add_set( input_values )
        add_json_file( arg )

    return options, input_values_sets, json_input_files
def read_xsection_measurement_results(category, channel):
    global path_to_JSON, variable, k_value, met_type
    normalised_xsection_unfolded = read_data_from_JSON(
        path_to_JSON + '/' + variable + '/xsection_measurement_results' +
        '/kv' + str(k_value) + '/' + category + '/normalised_xsection_' +
        channel + '_' + met_type + '.txt')
    h_normalised_xsection = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['TTJet_measured'], bin_edges[variable])
    h_normalised_xsection_unfolded = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['TTJet_unfolded'], bin_edges[variable])
    h_normalised_xsection_MADGRAPH = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['MADGRAPH'], bin_edges[variable])
    h_normalised_xsection_POWHEG = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['POWHEG'], bin_edges[variable])
    h_normalised_xsection_MCATNLO = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['MCATNLO'], bin_edges[variable])
    h_normalised_xsection_mathchingup = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['matchingup'], bin_edges[variable])
    h_normalised_xsection_mathchingdown = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['matchingdown'], bin_edges[variable])
    h_normalised_xsection_scaleup = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['scaleup'], bin_edges[variable])
    h_normalised_xsection_scaledown = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['scaledown'], bin_edges[variable])

    histograms_normalised_xsection_different_generators = {
        'measured': h_normalised_xsection,
        'unfolded': h_normalised_xsection_unfolded,
        'MADGRAPH': h_normalised_xsection_MADGRAPH,
        'POWHEG': h_normalised_xsection_POWHEG,
        'MCATNLO': h_normalised_xsection_MCATNLO
    }

    histograms_normalised_xsection_systematics_shifts = {
        'measured': h_normalised_xsection,
        'unfolded': h_normalised_xsection_unfolded,
        'matchingdown': h_normalised_xsection_mathchingdown,
        'matchingup': h_normalised_xsection_mathchingup,
        'scaledown': h_normalised_xsection_scaledown,
        'scaleup': h_normalised_xsection_scaleup
    }

    return histograms_normalised_xsection_different_generators, histograms_normalised_xsection_systematics_shifts
Exemplo n.º 43
0
def read_xsection_measurement_results_with_errors(channel):
    global path_to_JSON, variable, k_value, met_type
    category = 'central'

    normalised_xsection_unfolded = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + str(k_value) + '/' 
                                                       + category + '/normalised_xsection_' + channel + '_' + met_type + '.txt')
    
    normalised_xsection_measured_unfolded = {'measured':normalised_xsection_unfolded['TTJet_measured'],
                                            'unfolded':normalised_xsection_unfolded['TTJet_unfolded']}
    
    normalised_xsection_unfolded_with_errors = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + 
                                                                str(k_value) + '/' + category + '/normalised_xsection_' + 
                                                                channel + '_' + met_type + '_with_errors.txt')
    normalised_xsection_ttbar_theory_errors = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + 
                                                                str(k_value) + '/' + category + '/normalised_xsection_' + 
                                                                channel + '_' + met_type + '_ttbar_theory_errors.txt')
    normalised_xsection_MET_errors = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + 
                                                                str(k_value) + '/' + category + '/normalised_xsection_' + 
                                                                channel + '_' + met_type + '_MET_errors.txt')
    normalised_xsection_PDF_errors = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + 
                                                                str(k_value) + '/' + category + '/normalised_xsection_' + 
                                                                channel + '_' + met_type + '_PDF_errors.txt')
    normalised_xsection_other_errors = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + 
                                                                str(k_value) + '/' + category + '/normalised_xsection_' + 
                                                                channel + '_' + met_type + '_other_errors.txt')
    normalised_xsection_new_errors = read_data_from_JSON(path_to_JSON + '/xsection_measurement_results' + '/kv' + 
                                                                str(k_value) + '/' + category + '/normalised_xsection_' + 
                                                                channel + '_' + met_type + '_new_errors.txt')
    
    normalised_xsection_measured_unfolded.update({'measured_with_systematics':normalised_xsection_unfolded_with_errors['TTJet_measured'],
                                                'unfolded_with_systematics':normalised_xsection_unfolded_with_errors['TTJet_unfolded']})
    
    normalised_xsection_measured_errors = normalised_xsection_ttbar_theory_errors['TTJet_measured']
    normalised_xsection_measured_errors.update(normalised_xsection_PDF_errors['TTJet_measured'])
    normalised_xsection_measured_errors.update(normalised_xsection_MET_errors['TTJet_measured'])
    normalised_xsection_measured_errors.update(normalised_xsection_other_errors['TTJet_measured'])
    normalised_xsection_measured_errors.update(normalised_xsection_new_errors['TTJet_measured'])

    normalised_xsection_unfolded_errors = normalised_xsection_ttbar_theory_errors['TTJet_unfolded']
    normalised_xsection_unfolded_errors.update(normalised_xsection_PDF_errors['TTJet_unfolded'])
    normalised_xsection_unfolded_errors.update(normalised_xsection_MET_errors['TTJet_unfolded'])
    normalised_xsection_unfolded_errors.update(normalised_xsection_other_errors['TTJet_unfolded'])
    normalised_xsection_unfolded_errors.update(normalised_xsection_new_errors['TTJet_unfolded'])
    
    return normalised_xsection_measured_unfolded, normalised_xsection_measured_errors, normalised_xsection_unfolded_errors
    def __set_unfolding_histograms__(self):
        # at the moment only one file is supported for the unfolding input
        files = set([
            self.truth['file'], self.gen_vs_reco['file'], self.measured['file']
        ])
        if len(files) > 1:
            print "Currently not supported to have different files for truth, gen_vs_reco and measured"
            sys.exit()

        input_file = files.pop()
        t, m, r, _ = get_unfold_histogram_tuple(
            File(input_file),
            self.variable,
            self.channel,
            centre_of_mass=self.centre_of_mass_energy,
            ttbar_xsection=self.measurement_config.ttbar_xsection,
            luminosity=self.measurement_config.luminosity,
        )
        self.h_truth = t
        self.h_response = r
        self.h_measured = m

        data_file = self.data['file']
        if data_file.endswith('.root'):
            self.h_data = get_histogram_from_file(self.data['histogram'],
                                                  self.data['file'])
        elif data_file.endswith('.json') or data_file.endswith('.txt'):
            data_key = self.data['histogram']
            # assume configured bin edges
            edges = []
            if self.phaseSpace == 'FullPS':
                edges = bin_edges[self.variable]
            elif self.phaseSpace == 'VisiblePS':
                edges = bin_edges_vis[self.variable]
            json_input = read_data_from_JSON(data_file)
            if data_key == "":  # JSON file == histogram
                self.h_data = value_error_tuplelist_to_hist(json_input, edges)
            else:
                self.h_data = value_error_tuplelist_to_hist(
                    json_input[data_key], edges)
        else:
            print 'Unkown file extension', data_file.split('.')[-1]
def read_unfolded_xsections(channel):
    global path_to_JSON, variable, k_value, met_type, b_tag_bin
    TTJet_xsection_unfolded = {}
    for category in categories:
        normalised_xsections = read_data_from_JSON(
            path_to_JSON
            + "/"
            + variable
            + "/xsection_measurement_results"
            + "/kv"
            + str(k_value)
            + "/"
            + category
            + "/normalised_xsection_"
            + channel
            + "_"
            + met_type
            + ".txt"
        )
        TTJet_xsection_unfolded[category] = normalised_xsections["TTJet_unfolded"]
    return TTJet_xsection_unfolded
Exemplo n.º 46
0
def get_fitted_normalisation(variable, channel, path_to_JSON, category,
                             met_type):
    '''
    This function now gets the error on the fit correctly,
    so that it can be applied if the --normalise_to_fit option is used
    '''
    import config.variable_binning
    variable_bins_ROOT = config.variable_binning.variable_bins_ROOT
    fit_results = read_data_from_JSON(path_to_JSON + variable +
                                      '/fit_results/' + category +
                                      '/fit_results_' + channel + '_' +
                                      met_type + '.txt')

    N_fit_ttjet = [0, 0]
    N_fit_singletop = [0, 0]
    N_fit_vjets = [0, 0]
    N_fit_qcd = [0, 0]

    bins = variable_bins_ROOT[variable]
    for bin_i, _ in enumerate(bins):
        # central values
        N_fit_ttjet[0] += fit_results['TTJet'][bin_i][0]
        N_fit_singletop[0] += fit_results['SingleTop'][bin_i][0]
        N_fit_vjets[0] += fit_results['V+Jets'][bin_i][0]
        N_fit_qcd[0] += fit_results['QCD'][bin_i][0]

        # errors
        N_fit_ttjet[1] += fit_results['TTJet'][bin_i][1]
        N_fit_singletop[1] += fit_results['SingleTop'][bin_i][1]
        N_fit_vjets[1] += fit_results['V+Jets'][bin_i][1]
        N_fit_qcd[1] += fit_results['QCD'][bin_i][1]

    fitted_normalisation = {
        'TTJet': N_fit_ttjet,
        'SingleTop': N_fit_singletop,
        'V+Jets': N_fit_vjets,
        'QCD': N_fit_qcd
    }
    return fitted_normalisation
def read_normalised_xsection_measurement( category, channel ):
    global path_to_JSON, met_type, met_uncertainties_list, method
    filename = '{path}/{category}/normalised_xsection_{channel}_{method}.txt'
    if category in met_systematics_suffixes and ( variable in measurement_config.variables_no_met ) and not ('JES' in category or 'JER' in category):
        filename = filename.format(
                    path = path_to_JSON,
                    channel = channel,
                    category = 'central',
                    method = method,
                    )
    else:
        filename = filename.format(
                    path = path_to_JSON,
                    channel = channel,
                    category = category,
                    method = method)

    normalised_xsection = read_data_from_JSON( filename )
    
    measurement = normalised_xsection['TTJet_measured']
    measurement_unfolded = normalised_xsection['TTJet_unfolded']
    
    return measurement, measurement_unfolded
Exemplo n.º 48
0
def read_normalised_xsection_measurement(category, channel):
    global path_to_JSON, met_type, met_uncertainties_list, method
    filename = '{path}/{category}/normalised_xsection_{channel}_{method}.txt'
    if category in met_systematics_suffixes and (
            variable in measurement_config.variables_no_met
    ) and not ('JES' in category or 'JER' in category):
        filename = filename.format(
            path=path_to_JSON,
            channel=channel,
            category='central',
            method=method,
        )
    else:
        filename = filename.format(path=path_to_JSON,
                                   channel=channel,
                                   category=category,
                                   method=method)

    normalised_xsection = read_data_from_JSON(filename)

    measurement = normalised_xsection['TTJet_measured']
    measurement_unfolded = normalised_xsection['TTJet_unfolded']

    return measurement, measurement_unfolded
Exemplo n.º 49
0
    def fromJSON(JSON_file):
        src = fu.read_data_from_JSON(JSON_file)
        m = Measurement.fromDict(src)

        return m
Exemplo n.º 50
0
def read_xsection_measurement_results(path_to_JSON,
                                      variable,
                                      bin_edges,
                                      category,
                                      channel,
                                      k_values,
                                      met_type='patType1CorrectedPFMet',
                                      met_uncertainties=[]):

    filename = ''
    if category in met_uncertainties and variable == 'HT' and not 'JES' in category and not 'JER' in category:
        filename = path_to_JSON + '/xsection_measurement_results/' + \
            channel + '/central/normalised_xsection_' + met_type + '.txt'
    else:
        filename = path_to_JSON + '/xsection_measurement_results/' + channel + \
            '/' + category + '/normalised_xsection_' + met_type + '.txt'

    if channel == 'combined':
        filename = filename.replace('kv' + str(k_values[channel]), '')

    normalised_xsection_unfolded = read_data_from_JSON(filename)

    h_normalised_xsection = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['TTJet_measured'], bin_edges[variable])
    h_normalised_xsection_unfolded = value_error_tuplelist_to_hist(
        normalised_xsection_unfolded['TTJet_unfolded'], bin_edges[variable])

    histograms_normalised_xsection_different_generators = {
        'measured': h_normalised_xsection,
        'unfolded': h_normalised_xsection_unfolded
    }

    histograms_normalised_xsection_systematics_shifts = {
        'measured': h_normalised_xsection,
        'unfolded': h_normalised_xsection_unfolded
    }

    if category == 'central':
        # true distributions
        h_normalised_xsection_MADGRAPH = value_error_tuplelist_to_hist(
            normalised_xsection_unfolded['MADGRAPH'], bin_edges[variable])
        h_normalised_xsection_POWHEG_PYTHIA = value_error_tuplelist_to_hist(
            normalised_xsection_unfolded['POWHEG_PYTHIA'], bin_edges[variable])
        h_normalised_xsection_POWHEG_HERWIG = value_error_tuplelist_to_hist(
            normalised_xsection_unfolded['POWHEG_HERWIG'], bin_edges[variable])
        h_normalised_xsection_MCATNLO = value_error_tuplelist_to_hist(
            normalised_xsection_unfolded['MCATNLO'], bin_edges[variable])
        h_normalised_xsection_mathchingup = value_error_tuplelist_to_hist(
            normalised_xsection_unfolded['matchingup'], bin_edges[variable])
        h_normalised_xsection_mathchingdown = value_error_tuplelist_to_hist(
            normalised_xsection_unfolded['matchingdown'], bin_edges[variable])
        h_normalised_xsection_scaleup = value_error_tuplelist_to_hist(
            normalised_xsection_unfolded['scaleup'], bin_edges[variable])
        h_normalised_xsection_scaledown = value_error_tuplelist_to_hist(
            normalised_xsection_unfolded['scaledown'], bin_edges[variable])

        histograms_normalised_xsection_different_generators.update({
            'MADGRAPH':
            h_normalised_xsection_MADGRAPH,
            'POWHEG_PYTHIA':
            h_normalised_xsection_POWHEG_PYTHIA,
            'POWHEG_HERWIG':
            h_normalised_xsection_POWHEG_HERWIG,
            'MCATNLO':
            h_normalised_xsection_MCATNLO
        })

        histograms_normalised_xsection_systematics_shifts.update({
            'MADGRAPH':
            h_normalised_xsection_MADGRAPH,
            'matchingdown':
            h_normalised_xsection_mathchingdown,
            'matchingup':
            h_normalised_xsection_mathchingup,
            'scaledown':
            h_normalised_xsection_scaledown,
            'scaleup':
            h_normalised_xsection_scaleup
        })

        file_template = path_to_JSON + '/xsection_measurement_results/' + channel + \
            '/kv' + str(k_values[channel]) + '/' + \
            category + '/normalised_xsection_' + met_type
        if channel == 'combined':
            file_template = file_template.replace(
                'kv' + str(k_values[channel]), '')


#         normalised_xsection_unfolded_with_errors = read_data_from_JSON( file_template + '_with_errors.txt' )
        normalised_xsection_unfolded_with_errors_with_systematics_but_without_ttbar_theory = read_data_from_JSON(
            file_template +
            '_with_systematics_but_without_ttbar_theory_errors.txt')
        normalised_xsection_unfolded_with_errors_with_systematics_but_without_generator = read_data_from_JSON(
            file_template +
            '_with_systematics_but_without_generator_errors.txt')

        # a rootpy.Graph with asymmetric errors!
        h_normalised_xsection_with_systematics_but_without_ttbar_theory = value_errors_tuplelist_to_graph(
            normalised_xsection_unfolded_with_errors_with_systematics_but_without_ttbar_theory[
                'TTJet_measured'], bin_edges[variable])
        h_normalised_xsection_with_systematics_but_without_ttbar_theory_unfolded = value_errors_tuplelist_to_graph(
            normalised_xsection_unfolded_with_errors_with_systematics_but_without_ttbar_theory[
                'TTJet_unfolded'], bin_edges[variable])

        h_normalised_xsection_with_systematics_but_without_generator = value_errors_tuplelist_to_graph(
            normalised_xsection_unfolded_with_errors_with_systematics_but_without_generator[
                'TTJet_measured'], bin_edges[variable])
        h_normalised_xsection_with_systematics_but_without_generator_unfolded = value_errors_tuplelist_to_graph(
            normalised_xsection_unfolded_with_errors_with_systematics_but_without_generator[
                'TTJet_unfolded'], bin_edges[variable])

        histograms_normalised_xsection_different_generators[
            'measured_with_systematics'] = h_normalised_xsection_with_systematics_but_without_generator
        histograms_normalised_xsection_different_generators[
            'unfolded_with_systematics'] = h_normalised_xsection_with_systematics_but_without_generator_unfolded

        histograms_normalised_xsection_systematics_shifts[
            'measured_with_systematics'] = h_normalised_xsection_with_systematics_but_without_ttbar_theory
        histograms_normalised_xsection_systematics_shifts[
            'unfolded_with_systematics'] = h_normalised_xsection_with_systematics_but_without_ttbar_theory_unfolded

    return histograms_normalised_xsection_different_generators, histograms_normalised_xsection_systematics_shifts
def get_variable_from(variable='MET',
                      path_to_JSON='data/8TeV',
                      category='central',
                      signal='Higgs',
                      measurement_type='unfolded'):
    global met_type

    channel = 'electron'
    electron_results = read_data_from_JSON(
        path_to_JSON + '/' + variable + '/xsection_measurement_results/kv4/' +
        category + '/normalisation_' + channel + '_' + met_type + '.txt')
    electron_results_signal = read_data_from_JSON(
        path_to_JSON + '/' + variable + '/xsection_measurement_results/kv4/' +
        category + '/normalisation_' + channel + '_' + met_type + '_' +
        signal + '.txt')
    channel = 'muon'
    muon_results = read_data_from_JSON(path_to_JSON + '/' + variable +
                                       '/xsection_measurement_results/kv4/' +
                                       category + '/normalisation_' + channel +
                                       '_' + met_type + '.txt')
    muon_results_signal = read_data_from_JSON(
        path_to_JSON + '/' + variable + '/xsection_measurement_results/kv4/' +
        category + '/normalisation_' + channel + '_' + met_type + '_' +
        signal + '.txt')
    channel = 'combined'
    combined_results = read_data_from_JSON(
        path_to_JSON + '/' + variable + '/xsection_measurement_results/kv4/' +
        category + '/normalisation_' + channel + '_' + met_type + '.txt')
    combined_results_signal = read_data_from_JSON(
        path_to_JSON + '/' + variable + '/xsection_measurement_results/kv4/' +
        category + '/normalisation_' + channel + '_' + met_type + '_' +
        signal + '.txt')

    # we are only interested in the measured ttbar, unfolded TTbar for all categories
    # for_all_categories = ['TTJet_unfolded', 'TTJet_measured']
    # and the scale_up/down, matching up/down and the generators for central (all)
    electron_results_selected = {
        'Higgs_125': electron_results_signal['TTJet_' + measurement_type],
    }
    muon_results_selected = {
        'Higgs_125': muon_results_signal['TTJet_' + measurement_type],
    }
    combined_results_selected = {
        'Higgs_125': combined_results_signal['TTJet_' + measurement_type],
    }

    # note on systematics:
    # theta does not understand DATA systematics so they have to be migrated to MADGRAPH
    # there are two ways of doing so:
    # 1. calculate a scale factor based on data central measurement and MADPGRAPH
    # 2. calculate a scale factor based on data central and systematic measurement
    # then apply the scale factor (1- [a-b]/a) to MADGRAPH
    # The second approach seems more right as it takes into account the effect of the systematic instead of the difference between data and Madgraph
    # A way to test this is to calculate the scale factor for ttjet_matching up/down and compare it to the existing distributions
    if measurement_type == 'unfolded':
        electron_results_selected['TTJet'] = electron_results['MADGRAPH']
        muon_results_selected['TTJet'] = muon_results['MADGRAPH']
        combined_results_selected['TTJet'] = combined_results['MADGRAPH']

        # theta does not understand DATA with systematics
        if category == 'central':
            electron_results_selected['DATA'] = electron_results[
                'TTJet_unfolded']
            muon_results_selected['DATA'] = muon_results['TTJet_unfolded']
            combined_results_selected['DATA'] = combined_results[
                'TTJet_unfolded']
        else:  # now let's do the systematics
            # read central results
            channel = 'electron'
            tmp_category = 'central'
            tmp_electron_results = read_data_from_JSON(
                path_to_JSON + '/' + variable +
                '/xsection_measurement_results/kv4/' + tmp_category +
                '/normalisation_' + channel + '_' + met_type + '.txt')
            channel = 'muon'
            tmp_muon_results = read_data_from_JSON(
                path_to_JSON + '/' + variable +
                '/xsection_measurement_results/kv4/' + tmp_category +
                '/normalisation_' + channel + '_' + met_type + '.txt')
            channel = 'combined'
            tmp_combined_results = read_data_from_JSON(
                path_to_JSON + '/' + variable +
                '/xsection_measurement_results/kv4/' + tmp_category +
                '/normalisation_' + channel + '_' + met_type + '.txt')

            electron_central = tmp_electron_results['TTJet_unfolded']
            muon_central = tmp_muon_results['TTJet_unfolded']
            combined_central = tmp_combined_results['TTJet_unfolded']
            # set systematics
            electron_systematic = electron_results['TTJet_unfolded']
            muon_systematic = muon_results['TTJet_unfolded']
            combined_systematic = combined_results['TTJet_unfolded']
            # calculate scale factors
            electron_results_selected['TTJet'] = morph_systematic(
                electron_central, electron_systematic,
                electron_results_selected['TTJet'])
            muon_results_selected['TTJet'] = morph_systematic(
                muon_central, muon_systematic, muon_results_selected['TTJet'])
            combined_results_selected['TTJet'] = morph_systematic(
                combined_central, combined_systematic,
                combined_results_selected['TTJet'])

    else:
        if category == 'JES_up':
            met_type += 'JetEnUp'
        elif category == 'JES_down':
            met_type += 'JetEnDown'
        # read initial values
        channel = 'electron'
        electron_initial = read_data_from_JSON(path_to_JSON + '/' + variable +
                                               '/fit_results/' + category +
                                               '/initial_values_' + channel +
                                               '_' + met_type + '.txt')
        electron_fit = read_data_from_JSON(path_to_JSON + '/' + variable +
                                           '/fit_results/' + category +
                                           '/fit_results_' + channel + '_' +
                                           met_type + '.txt')
        channel = 'muon'
        muon_initial = read_data_from_JSON(path_to_JSON + '/' + variable +
                                           '/fit_results/' + category +
                                           '/initial_values_' + channel + '_' +
                                           met_type + '.txt')
        muon_fit = read_data_from_JSON(path_to_JSON + '/' + variable +
                                       '/fit_results/' + category +
                                       '/fit_results_' + channel + '_' +
                                       met_type + '.txt')

        electron_results_selected['TTJet'] = electron_results['TTJet_measured']
        muon_results_selected['TTJet'] = muon_results['TTJet_measured']

        electron_results_selected['SingleTop'] = electron_fit['SingleTop']
        muon_results_selected['SingleTop'] = muon_fit['SingleTop']

        electron_results_selected['QCD'] = electron_fit['QCD']
        muon_results_selected['QCD'] = muon_fit['QCD']
        # theta does not understand DATA with systematics
        if category == 'central':
            electron_results_selected['DATA'] = electron_initial['data']
            muon_results_selected['DATA'] = muon_initial['data']

        electron_results_selected['VJets'] = electron_fit['V+Jets']
        muon_results_selected['VJets'] = muon_fit['V+Jets']

        # reset met for JES
        if category == 'JES_up':
            met_type = met_type.replace('JetEnUp', '')
        elif category == 'JES_down':
            met_type = met_type.replace('JetEnDown', '')

    return electron_results_selected, muon_results_selected, combined_results_selected
def read_xsection_measurement_results( category, channel ):
    global path_to_JSON, variable, met_type, phase_space, method

    file_template = '{path}/{category}/{name}_{channel}_{method}{suffix}.txt'
    filename = file_template.format(
                path = path_to_JSON,
                category = category,
                name = 'normalised_xsection',
                channel = channel,
                method = method,
                suffix = '',
                )

    xsec_04_log.debug('Reading file {0}'.format(filename))
    normalised_xsection_unfolded = read_data_from_JSON( filename )
    edges = bin_edges[variable]
    if phase_space == 'VisiblePS':
        edges = bin_edges_vis[variable]
    h_normalised_xsection = value_error_tuplelist_to_hist( normalised_xsection_unfolded['TTJet_measured'], edges )
    h_normalised_xsection_unfolded = value_error_tuplelist_to_hist( normalised_xsection_unfolded['TTJet_unfolded'], edges )


    histograms_normalised_xsection_different_generators = {'measured':h_normalised_xsection,
                                                           'unfolded':h_normalised_xsection_unfolded}

    histograms_normalised_xsection_systematics_shifts = {'measured':h_normalised_xsection,
                                                         'unfolded':h_normalised_xsection_unfolded}

    if category == 'central':
        # true distributions
        h_normalised_xsection_powhegPythia8 = value_error_tuplelist_to_hist( normalised_xsection_unfolded['powhegPythia8'], edges )
        h_normalised_xsection_amcatnlo = value_error_tuplelist_to_hist( normalised_xsection_unfolded['amcatnlo'], edges )
        h_normalised_xsection_madgraphMLM = value_error_tuplelist_to_hist( normalised_xsection_unfolded['madgraphMLM'], edges )
        h_normalised_xsection_amcatnloHerwigpp = value_error_tuplelist_to_hist( normalised_xsection_unfolded['amcatnlo_HERWIG'], edges )

        h_normalised_xsection_scaleup = value_error_tuplelist_to_hist( normalised_xsection_unfolded['scaleup'], edges )
        h_normalised_xsection_scaledown = value_error_tuplelist_to_hist( normalised_xsection_unfolded['scaledown'], edges )
        h_normalised_xsection_massup = value_error_tuplelist_to_hist( normalised_xsection_unfolded['massup'], edges )
        h_normalised_xsection_massdown = value_error_tuplelist_to_hist( normalised_xsection_unfolded['massdown'], edges )

        histograms_normalised_xsection_different_generators.update( {
                                                                     'powhegPythia8':h_normalised_xsection_powhegPythia8,
                                                                     'amcatnlo':h_normalised_xsection_amcatnlo,
                                                                     'madgraphMLM':h_normalised_xsection_madgraphMLM,
                                                                     'amcatnlo_HERWIG':h_normalised_xsection_amcatnloHerwigpp,
                                                                })

        histograms_normalised_xsection_systematics_shifts.update( {'powhegPythia8':h_normalised_xsection_powhegPythia8,
                                                                  'scaledown': h_normalised_xsection_scaledown,
                                                                  'scaleup': h_normalised_xsection_scaleup,
                                                                  'massdown': h_normalised_xsection_massdown,
                                                                  'massup': h_normalised_xsection_massup
                                                                  })

        filename = file_template.format(
                path = path_to_JSON,
                category = category,
                name = 'normalised_xsection',
                channel = channel,
                method = method,
                suffix = '_with_errors',
                )

        normalised_xsection_unfolded_with_errors = read_data_from_JSON( filename )
        xsec_04_log.debug('Reading file {0}'.format(filename))
#         filename = file_template.format(
#                 path = path_to_JSON,
#                 category = category,
#                 name = 'normalised_xsection',
#                 channel = channel,
#                 method = method,
#                 suffix = '_with_systematics_but_without_generator_errors',
#                 )
        ### normalised_xsection_unfolded_with_errors_with_systematics_but_without_ttbar_theory = read_data_from_JSON( file_template + '_with_systematics_but_without_ttbar_theory_errors.txt' )
#         normalised_xsection_unfolded_with_errors_with_systematics_but_without_generator = normalised_xsection_unfolded_with_errors

        # a rootpy.Graph with asymmetric errors!
        ### h_normalised_xsection_with_systematics_but_without_ttbar_theory = value_errors_tuplelist_to_graph(
        ###                                                         normalised_xsection_unfolded_with_errors_with_systematics_but_without_ttbar_theory['TTJet_measured'],
        ###                                                         edges )
        ### h_normalised_xsection_with_systematics_but_without_ttbar_theory_unfolded = value_errors_tuplelist_to_graph(
        ###                                                         normalised_xsection_unfolded_with_errors_with_systematics_but_without_ttbar_theory['TTJet_unfolded'],
        ###                                                         edges )

        h_normalised_xsection_unfolded_with_errors = value_errors_tuplelist_to_graph(
                                                                normalised_xsection_unfolded_with_errors['TTJet_measured'],
                                                                edges )
        h_normalised_xsection_unfolded_with_errors_unfolded = value_errors_tuplelist_to_graph(
                                                                normalised_xsection_unfolded_with_errors['TTJet_unfolded'],
                                                                edges )


        # histograms_normalised_xsection_different_generators['measured_with_systematics'] = h_normalised_xsection_with_systematics_but_without_ttbar_theory
        # histograms_normalised_xsection_different_generators['unfolded_with_systematics'] = h_normalised_xsection_with_systematics_but_without_ttbar_theory_unfolded
        histograms_normalised_xsection_different_generators['measured_with_systematics'] = h_normalised_xsection_unfolded_with_errors
        histograms_normalised_xsection_different_generators['unfolded_with_systematics'] = h_normalised_xsection_unfolded_with_errors_unfolded

        histograms_normalised_xsection_systematics_shifts['measured_with_systematics'] = h_normalised_xsection_unfolded_with_errors
        histograms_normalised_xsection_systematics_shifts['unfolded_with_systematics'] = h_normalised_xsection_unfolded_with_errors_unfolded

    return histograms_normalised_xsection_different_generators, histograms_normalised_xsection_systematics_shifts
Exemplo n.º 53
0
def get_data_histogram( channel, variable, met_type ):
    fit_result_input = 'data/M3_angle_bl/13TeV/%(variable)s/fit_results/central/fit_results_%(channel)s_%(met_type)s.txt'
    fit_results = read_data_from_JSON( fit_result_input % {'channel': channel, 'variable': variable, 'met_type':met_type} )
    fit_data = fit_results['TTJet']
    h_data = value_error_tuplelist_to_hist( fit_data, bin_edges[variable] )
    return h_data
def debug_last_bin():
    '''
        For debugging why the last bin in the problematic variables deviates a
        lot in _one_ of the channels only.
    '''
    file_template = '/hdfs/TopQuarkGroup/run2/dpsData/'
    file_template += 'data/normalisation/background_subtraction/13TeV/'
    file_template += '{variable}/VisiblePS/central/'
    file_template += 'normalised_xsection_{channel}_RooUnfoldSvd{suffix}.txt'
    problematic_variables = ['HT', 'MET', 'NJets', 'lepton_pt']

    for variable in problematic_variables:
        results = {}
        Result = namedtuple('Result',
                            ['before_unfolding', 'after_unfolding', 'model'])
        for channel in ['electron', 'muon', 'combined']:
            input_file_data = file_template.format(
                variable=variable,
                channel=channel,
                suffix='_with_errors',
            )
            input_file_model = file_template.format(
                variable=variable,
                channel=channel,
                suffix='',
            )
            data = read_data_from_JSON(input_file_data)
            data_model = read_data_from_JSON(input_file_model)
            before_unfolding = data['TTJet_measured_withoutFakes']
            after_unfolding = data['TTJet_unfolded']

            model = data_model['powhegPythia8']

            # only use the last bin
            h_before_unfolding = value_errors_tuplelist_to_graph(
                [before_unfolding[-1]], bin_edges_vis[variable][-2:])
            h_after_unfolding = value_errors_tuplelist_to_graph(
                [after_unfolding[-1]], bin_edges_vis[variable][-2:])
            h_model = value_error_tuplelist_to_hist(
                [model[-1]], bin_edges_vis[variable][-2:])

            r = Result(before_unfolding, after_unfolding, model)
            h = Result(h_before_unfolding, h_after_unfolding, h_model)
            results[channel] = (r, h)

        models = {'POWHEG+PYTHIA': results['combined'][1].model}
        h_unfolded = [
            results[channel][1].after_unfolding
            for channel in ['electron', 'muon', 'combined']
        ]
        tmp_hists = spread_x(h_unfolded, bin_edges_vis[variable][-2:])
        measurements = {}
        for channel, hist in zip(['electron', 'muon', 'combined'], tmp_hists):
            value = results[channel][0].after_unfolding[-1][0]
            error = results[channel][0].after_unfolding[-1][1]
            label = '{c_label} ({value:1.2g} $\pm$ {error:1.2g})'.format(
                c_label=channel,
                value=value,
                error=error,
            )
            measurements[label] = hist

        properties = Histogram_properties()
        properties.name = 'normalised_xsection_compare_channels_{0}_{1}_last_bin'.format(
            variable, channel)
        properties.title = 'Comparison of channels'
        properties.path = 'plots'
        properties.has_ratio = True
        properties.xerr = False
        properties.x_limits = (bin_edges_vis[variable][-2],
                               bin_edges_vis[variable][-1])
        properties.x_axis_title = variables_latex[variable]
        properties.y_axis_title = r'$\frac{1}{\sigma}  \frac{d\sigma}{d' + \
            variables_latex[variable] + '}$'
        properties.legend_location = (0.95, 0.40)
        if variable == 'NJets':
            properties.legend_location = (0.97, 0.80)
        properties.formats = ['png']

        compare_measurements(models=models,
                             measurements=measurements,
                             show_measurement_errors=True,
                             histogram_properties=properties,
                             save_folder='plots/',
                             save_as=properties.formats)
Exemplo n.º 55
0
        #         muon_file = path_to_JSON + '/'+category+'/initial_normalisation_muon_' + met_type + '.txt'
        elif category != 'central':
            # electron_file = path_to_JSON + '/'+category+'/initial_normalisation_electron_' + met_type + '.txt'
            # muon_file = path_to_JSON + '/'+category+'/initial_normalisation_muon_' + met_type + '.txt'
            # electron_file = path_to_JSON + '/central/normalisation_electron_' + translate_options[options.metType] + '.txt'
            # muon_file = path_to_JSON + '/central/normalisation_muon_' + translate_options[options.metType] + '.txt'
            electron_file = path_to_JSON + '/' + category + '/normalisation_electron_' + met_type + '.txt'
            muon_file = path_to_JSON + '/' + category + '/normalisation_muon_' + met_type + '.txt'

        fit_results_electron = None
        fit_results_muon = None

        if category == 'Muon_up' or category == 'Muon_down':
            # fit_results_electron = read_data_from_JSON( path_to_JSON + '/central/initial_normalisation_electron_' + met_type + '.txt' )
            fit_results_electron = read_data_from_JSON(
                path_to_JSON + '/central/normalisation_electron_' + met_type +
                '.txt')
            fit_results_muon = read_data_from_JSON(muon_file)
        elif category == 'Electron_up' or category == 'Electron_down':
            fit_results_electron = read_data_from_JSON(electron_file)
            # fit_results_muon = read_data_from_JSON( path_to_JSON + '/central/initial_normalisation_muon_' + met_type + '.txt' )
            fit_results_muon = read_data_from_JSON(
                path_to_JSON + '/central/normalisation_muon_' + met_type +
                '.txt')
        else:
            fit_results_electron = read_data_from_JSON(electron_file)
            fit_results_muon = read_data_from_JSON(muon_file)
        fit_results_combined = combine_complex_results(fit_results_electron,
                                                       fit_results_muon)
        TTJet_fit_results_electron = fit_results_electron['TTJet']
        TTJet_fit_results_muon = fit_results_muon['TTJet']
        #         electron_file = path_to_JSON + '/'+category+'/initial_normalisation_electron_' + met_type + '.txt'
        #         muon_file = path_to_JSON + '/'+category+'/initial_normalisation_muon_' + met_type + '.txt'
        elif category != 'central':
                # electron_file = path_to_JSON + '/'+category+'/initial_normalisation_electron_' + met_type + '.txt'
                # muon_file = path_to_JSON + '/'+category+'/initial_normalisation_muon_' + met_type + '.txt'
                # electron_file = path_to_JSON + '/central/normalisation_electron_' + translate_options[options.metType] + '.txt'
                # muon_file = path_to_JSON + '/central/normalisation_muon_' + translate_options[options.metType] + '.txt'
                electron_file = path_to_JSON + '/' + category + '/normalisation_electron_' + met_type + '.txt'
                muon_file = path_to_JSON + '/' + category + '/normalisation_muon_' + met_type + '.txt'    

        fit_results_electron = None
        fit_results_muon = None
        
        if category == 'Muon_up' or category == 'Muon_down':
            # fit_results_electron = read_data_from_JSON( path_to_JSON + '/central/initial_normalisation_electron_' + met_type + '.txt' )
            fit_results_electron = read_data_from_JSON( path_to_JSON + '/central/normalisation_electron_' + met_type + '.txt' )
            fit_results_muon = read_data_from_JSON( muon_file )
        elif category == 'Electron_up' or category == 'Electron_down':
            fit_results_electron = read_data_from_JSON( electron_file )
            # fit_results_muon = read_data_from_JSON( path_to_JSON + '/central/initial_normalisation_muon_' + met_type + '.txt' )
            fit_results_muon = read_data_from_JSON( path_to_JSON + '/central/normalisation_muon_' + met_type + '.txt' )
        else:
            fit_results_electron = read_data_from_JSON( electron_file )
            fit_results_muon = read_data_from_JSON( muon_file )
        fit_results_combined = combine_complex_results(fit_results_electron, fit_results_muon)
        TTJet_fit_results_electron = fit_results_electron['TTJet']
        TTJet_fit_results_muon = fit_results_muon['TTJet']
        TTJet_fit_results_combined = fit_results_combined['TTJet']

    #     # change back to original MET type for the unfolding
        met_type = translate_options[options.metType]
    for category in categories:
        #Setting up systematic MET for JES up/down samples
        met_type = translateOptions[options.metType]
        if category == 'JES_up':
            met_type += 'JetEnUp'
            if met_type == 'PFMETJetEnUp':
                met_type = 'patPFMetJetEnUp'
        elif category == 'JES_down':
            met_type += 'JetEnDown'
            if met_type == 'PFMETJetEnDown':
                met_type = 'patPFMetJetEnDown'

        #read fit results from JSON
        TTJet_fit_results_electron = read_data_from_JSON(
            path_to_JSON + '/' + variable + '/fit_results/' + category +
            '/fit_results_electron_' + met_type + '.txt')['TTJet']
        TTJet_fit_results_muon = read_data_from_JSON(path_to_JSON + '/' +
                                                     variable +
                                                     '/fit_results/' +
                                                     category +
                                                     '/fit_results_muon_' +
                                                     met_type +
                                                     '.txt')['TTJet']

        #change back to original MET type for the unfolding
        met_type = translateOptions[options.metType]
        #ad-hoc switch for PFMET -> patMETsPFlow
        if met_type == 'PFMET':
            met_type = 'patMETsPFlow'