def main():
set_root_defaults()
options, _ = parse_arguments()
variable = 'ST'
config_7TeV = XSectionConfig(7)
config_8TeV = XSectionConfig(8)
path_to_JSON_7TeV = options.path + '/7TeV/' + variable + '/'
path_to_JSON_8TeV = options.path + '/8TeV/' + variable + '/'
# we need the generators
# and the central samples + errors
results_7TeV, _ = read_xsection_measurement_results(
path_to_JSON_7TeV,
variable,
bin_edges,
category='central',
channel='combined',
k_values={'combined': config_7TeV.k_values_combined})
results_8TeV, _ = read_xsection_measurement_results(
path_to_JSON_8TeV,
variable,
bin_edges,
category='central',
channel='combined',
k_values={'combined': config_8TeV.k_values_combined})
plot_results(results_7TeV, results_8TeV, variable)
def get_histograms( variable, options ):
config = XSectionConfig( 13 )
path_electron = ''
path_muon = ''
path_combined = ''
histogram_name = ''
if options.visiblePhaseSpace:
histogram_name = 'responseVis_without_fakes'
else :
histogram_name = 'response_without_fakes'
if variable == 'HT':
path_electron = 'unfolding_HT_analyser_electron_channel/%s' % histogram_name
path_muon = 'unfolding_HT_analyser_muon_channel/%s' % histogram_name
path_combined = 'unfolding_HT_analyser_COMBINED_channel/%s' % histogram_name
else :
path_electron = 'unfolding_%s_analyser_electron_channel_patType1CorrectedPFMet/%s' % ( variable, histogram_name )
path_muon = 'unfolding_%s_analyser_muon_channel_patType1CorrectedPFMet/%s' % ( variable, histogram_name )
path_combined = 'unfolding_%s_analyser_COMBINED_channel_patType1CorrectedPFMet/%s' % ( variable, histogram_name )
histogram_information = [
{'file': config.unfolding_central_raw,
'CoM': 13,
'path':path_electron,
'channel':'electron'},
{'file':config.unfolding_central_raw,
'CoM': 13,
'path':path_muon,
'channel':'muon'},
]
if options.combined:
histogram_information = [
{'file': config.unfolding_central_raw,
'CoM': 13,
'path': path_combined,
'channel':'combined'},
]
for histogram in histogram_information:
f = File( histogram['file'] )
# scale to lumi
# nEvents = f.EventFilter.EventCounter.GetBinContent( 1 ) # number of processed events
# config = XSectionConfig( histogram['CoM'] )
# lumiweight = config.ttbar_xsection * config.new_luminosity / nEvents
lumiweight = 1
histogram['hist'] = f.Get( histogram['path'] ).Clone()
histogram['hist'].Scale( lumiweight )
# change scope from file to memory
histogram['hist'].SetDirectory( 0 )
f.close()
return histogram_information
def main(options, args):
config = XSectionConfig(options.CoM)
variables = ['MET', 'HT', 'ST', 'WPT']
channels = ['electron', 'muon', 'combined']
m_file = 'normalised_xsection_patType1CorrectedPFMet.txt'
m_with_errors_file = 'normalised_xsection_patType1CorrectedPFMet_with_errors.txt'
path_template = args[0]
output_file = 'measurement_{0}TeV.root'.format(options.CoM)
f = File(output_file, 'recreate')
for channel in channels:
d = f.mkdir(channel)
d.cd()
for variable in variables:
dv = d.mkdir(variable)
dv.cd()
if channel == 'combined':
path = path_template.format(variable=variable,
channel=channel,
centre_of_mass_energy=options.CoM)
else:
kv = channel + \
'/kv{0}/'.format(config.k_values[channel][variable])
path = path_template.format(variable=variable,
channel=kv,
centre_of_mass_energy=options.CoM)
m = read_data_from_JSON(path + '/' + m_file)
m_with_errors = read_data_from_JSON(path + '/' +
m_with_errors_file)
for name, result in m.items():
h = make_histogram(result, bin_edges[variable])
h.SetName(name)
h.write()
for name, result in m_with_errors.items():
if not 'TTJet' in name:
continue
h = make_histogram(result, bin_edges[variable])
h.SetName(name + '_with_syst')
h.write()
dv.write()
d.cd()
d.write()
f.write()
f.close()
def __init__(self, input_values):
self.centre_of_mass_energy = input_values['centre-of-mass energy']
self.measurement_config = XSectionConfig(self.centre_of_mass_energy)
self.channel = input_values['channel']
self.variable = input_values['variable']
self.phaseSpace = input_values['phaseSpace']
self.output_folder = input_values['output_folder']
self.output_format = input_values['output_format']
self.truth = input_values['truth']
self.gen_vs_reco = input_values['gen_vs_reco']
self.measured = input_values['measured']
self.data = input_values['data']
# optional
if input_values.has_key('n_tau_scan_points'):
self.n_tau_scan_points = input_values['n_tau_scan_points']
if input_values.has_key('n_toy'):
self.n_toy = input_values['n_toy']
self.__set_unfolding_histograms__()
parser.add_option(
"-c",
"--centre-of-mass-energy",
dest="CoM",
default=13,
type=int,
help="set the centre of mass energy for analysis. Default = 13 [TeV]")
parser.add_option(
"--category",
dest="category",
default='central',
help="set the category to take the fit results from (default: central)"
)
(options, args) = parser.parse_args()
measurement_config = XSectionConfig(options.CoM)
# caching of variables for shorter access
translate_options = measurement_config.translate_options
# Input and output
path_to_JSON = '%s/%dTeV/' % (options.path,
measurement_config.centre_of_mass_energy)
output_folder = '%s/before_fit/%dTeV/' % (
options.output_folder, measurement_config.centre_of_mass_energy)
make_folder_if_not_exists(output_folder)
# Central or whatever
category = options.category
# Input files for each group of samples
histogram_files = {
8 : '/hdfs/TopQuarkGroup/results/histogramfiles/AN-14-071_5th_draft/8TeV/unfolding/unfolding_TTJets_8TeV_asymmetric.root',
13 : 'unfolding/13TeV/unfolding_TTJets_13TeV_asymmetric.root',
}
met_type = 'patType1CorrectedPFMet'
# ST and HT have the problem of the overflow bin in the truth/response matrix
# 7 input bins and 8 output bins (includes 1 overflow bin)
variables = [
# 'MET',
# 'WPT',
# 'MT' ,
'ST',
'HT'
]
centre_of_mass = 13
measurement_config = XSectionConfig(centre_of_mass)
ttbar_xsection = measurement_config.ttbar_xsection
luminosity = measurement_config.luminosity
input_file = File(input_files[centre_of_mass])
# taus_from_global_correlaton = {}
# taus_from_L_shape = {}
for variable in variables:
# taus_from_global_correlaton[channel] = {}
# taus_from_L_shape[channel] = {}
for channel in ['electron', 'muon']:
if channel == 'electron':
used_k = measurement_config.k_values_electron[variable]
if channel == 'muon':
used_k = measurement_config.k_values_muon[variable]
dest="listJobs",
action='store_true',
default=False,
help="Just list the jobs to run and the total number of jobs")
(options, _) = parser.parse_args()
if options.jobNumber < 0:
print "Choose a job number >= 0"
sys.exit()
if options.com != 7 and options.com != 8:
print "Centre of mass must be 7 or 8 TeV"
print "You've chosen", options.com
sys.exit()
config = XSectionConfig(options.com)
sample_summations = None
if options.com == 7:
sample_summations = sample_summations_7TeV
elif options.com == 8:
sample_summations = sample_summations_8TeV
#Can not output to DICE, and since we are now working with input files on /hdfs, need to find somewhere to output the merged files
#this script creates
# first get current working directory
current_working_directory = os.getcwd()
path_to_AN_folder = config.path_to_files
# Where you want files to end up on hdfs
# Because you can't merge directly to hdfs at the moment,
# have to merge somewhere else then cp/mv/rsync file to hdfs
import json
from config import XSectionConfig
from config.variable_binning import bin_edges, bin_edges_vis
from tools.file_utilities import make_folder_if_not_exists
com = 13
fitVars = "M3_angle_bl"
config = XSectionConfig( com )
make_folder_if_not_exists('config/unfolding/FullPS/')
make_folder_if_not_exists('config/unfolding/VisiblePS/')
for channel in config.analysis_types.keys():
for variable in bin_edges.keys():
histogramTemplate = "unfolding_%s_analyser_%s_channel" % ( variable, channel )
outputJson = {
"output_folder": "plots/%sTeV/unfolding_tests/FullPS" % com,
"output_format": ["png", "pdf"],
"centre-of-mass energy" : com,
"channel": "%s" % channel,
"variable": "%s" % variable,
"phaseSpace" : "FullPS",
"truth" : {
"file" : "%s" % config.unfolding_central,
# "histogram": "%s/truth" % ( histogramTemplate ),
},
"gen_vs_reco" : {
"file" : "%s" % config.unfolding_central,
# "histogram": "%s/response_without_fakes" % ( histogramTemplate ),
def table(result_dict, use_current_k_values=False, style='simple'):
'''
result_dict has the form
{
centre-of-mass energy :
{
variable :
{
channel : (k_results, tau_results)
}
}
}
<reg>_results are of the form:
best_<reg>, best_<reg>_rho, <reg>s, rhos
'''
for com in result_dict.keys():
# step 1: group the results by centre of mass energy
headers = []
if use_current_k_values:
headers = [
'Variable', 'current k', 'closest tau', 'best tau', 'best k'
]
else:
headers = [
'Variable', 'best k', 'rho (best k)', 'best tau',
'rho (best tau)'
]
data = []
configOutputElectron = {}
configOutputMuon = {}
configOutputCombined = {}
measurement_config = XSectionConfig(com)
for variable in result_dict[com].keys():
has_both_channels = len(result_dict[com][variable]) == 3
# step 2: if have electron and muon channel, group them: electron (muon)
if has_both_channels:
electron_results = result_dict[com][variable]['electron']
muon_results = result_dict[com][variable]['muon']
combined_results = result_dict[com][variable]['combined']
configOutputElectron[variable] = electron_results[1][0]
configOutputMuon[variable] = muon_results[1][0]
configOutputCombined[variable] = combined_results[1][0]
entry = []
if use_current_k_values:
electron_set = get_k_tau_set(measurement_config,
'electron', variable,
electron_results)
muon_set = get_k_tau_set(measurement_config, 'muon',
variable, muon_results)
combined_set = get_k_tau_set(measurement_config,
'combined', variable,
combined_results)
entry = [
variable,
'%d (%d)' %
(electron_set[0], muon_set[0], combined_set[0]),
'%.1f (%.1f)' %
(electron_set[1], muon_set[1], combined_set[1]),
'%.1f (%.1f)' %
(electron_set[2], muon_set[2], combined_set[2]),
'%d (%d)' %
(electron_set[3], muon_set[3], combined_set[3]),
]
else:
entry = [
variable,
'%d %d %d' %
(electron_results[0][0], muon_results[0][0],
combined_results[0][0]),
'%.1f %.1f %.1f' %
(electron_results[0][1], muon_results[0][1],
combined_results[0][1]),
'%.1f %.1f %.1f' %
(electron_results[1][0], muon_results[1][0],
combined_results[1][0]),
'%.1f %.1f %.1f' %
(electron_results[1][1], muon_results[1][1],
combined_results[1][1]),
]
data.append(entry)
else:
channel = result_dict[com][variable].keys()[0]
results = result_dict[com][variable][channel]
print channel
if channel == 'electron':
configOutputElectron[variable] = results[1][0]
elif channel == 'muon':
configOutputMuon[variable] = results[1][0]
else:
configOutputCombined[variable] = results[1][0]
if use_current_k_values:
result_set = get_k_tau_set(measurement_config, channel,
variable, results)
entry = [
variable,
'%d' % result_set[0],
'%.1f' % result_set[1],
'%.1f' % result_set[2],
'%d' % result_set[3],
]
else:
entry = [
variable,
'%d' % results[0][0],
'%.1f' % results[0][1],
'%.1f' % results[1][0],
'%.1f' % results[1][1],
]
data.append(entry)
print '\nOutput for __init__\n'
print configOutputElectron
print configOutputMuon
print configOutputCombined
print 'Electron'
for var in configOutputElectron:
print '"%s" : %s,' % (var, configOutputElectron[var])
print '\n'
print 'Muon'
for var in configOutputMuon:
print '"%s" : %s,' % (var, configOutputMuon[var])
print '\n'
print 'Combined'
for var in configOutputCombined:
print '"%s" : %s,' % (var, configOutputCombined[var])
print '\n'
table = PrintTable(data, headers)
print 'Printing table for sqrt(s) = %d TeV' % com
if style == 'simple':
print table.simple()
elif style == 'latex':
print table.latex()
elif style == 'twiki':
print table.twiki()
else:
print 'unknown printing style "%s"' % style
def setUp(self):
self.config_7TeV = XSectionConfig(centre_of_mass_energy=7)
self.config_8TeV = XSectionConfig(centre_of_mass_energy=8)
self.config_13TeV = XSectionConfig(centre_of_mass_energy=13)
from config.summations_8TeV import sample_summations
from config import XSectionConfig
from tools.file_utilities import make_folder_if_not_exists
from tools.file_utilities import merge_ROOT_files
import os
import subprocess
import time
new_files = []
config_8TeV = XSectionConfig(8)
#Can not output to DICE, and since we are now working with input files on /hdfs, need to find somewhere to output the merged files
#this script creates
#So, create folders in working directory with structure: AN-XX-XXX_Xth_draft/XTeV/<central/BJet_up/Light_Jet_up/etc. for all categories>
#NOTE: YOU WILL THEN HAVE TO MOVE THESE MERGED FILES MANUALLY TO THE APPROPRIATE LOCATION IN /hdfs/TopQuarkGroup/results/histogramfiles/...
# first get current working directory
current_working_directory = os.getcwd()
path_to_AN_folder = config_8TeV.path_to_files
# change path from /hdfs to current working directory
path_to_AN_folder = path_to_AN_folder.replace(
"/hdfs/TopQuarkGroup/results/histogramfiles", current_working_directory)
#loop through all categories (e.g. central, BJet_up, LightJet_up, etc....) and make folder
for category in config_8TeV.categories_and_prefixes.keys():
make_folder_if_not_exists(path_to_AN_folder + "/" + category)
# merge generator systematics histogram files
for sample, input_samples in sample_summations.iteritems():
if not sample in [
def main():
parser = OptionParser(__doc__)
parser.add_option(
"-v",
"--variable",
dest="variable",
default='MET',
help="set the variable to analyse (MET, HT, ST, MT, WPT)")
parser.add_option(
"-s",
"--centre-of-mass-energy",
dest="CoM",
default=8,
help="set the centre of mass energy for analysis. Default = 8 [TeV]",
type=int)
parser.add_option("-o",
"--output",
dest="output_folder",
default='plots/unfolding_pulls',
help="output folder for unfolding pull plots")
parser.add_option("-c",
"--channel",
type='string',
dest="channel",
default='combined',
help="channel to be analysed: electron|muon|combined")
parser.add_option(
"-k",
"--k_value",
type='int',
dest="k_value",
default=-1,
help=
"k-value used in SVD unfolding, only for categorisation purpose at this stage"
)
parser.add_option("--tau",
type='float',
dest="tau_value",
default=-1.,
help="tau-value for SVD unfolding")
(options, args) = parser.parse_args()
# measurement_config = XSectionConfig(options.CoM)
if len(args) == 0:
print('No input files specified.')
print('Run script with "-h" for usage')
sys.exit(-1)
files = args
centre_of_mass = options.CoM
variable = options.variable
channel = options.channel
config = XSectionConfig(centre_of_mass)
k_value = get_k_value(options.k_value, config, channel, variable)
tau_value = get_tau_value(options.tau_value, config, channel, variable)
output_folder = '{option_output}/{centre_of_mass}TeV/{variable}/{channel}/'
output_folder = output_folder.format(option_output=options.output_folder,
centre_of_mass=centre_of_mass,
variable=variable,
channel=channel)
make_folder_if_not_exists(output_folder)
output_formats = ['pdf']
bins = array('d', bin_edges[variable])
nbins = len(bins) - 1
msg = 'Producing unfolding pull plots for {0} variable, channel: {1}'
print(msg.format(variable, channel))
value = get_value_title(k_value, tau_value)
print('Using {0}'.format(value))
print('Output folder: {0}'.format(output_folder))
pulls = get_data(files, subset='pull')
fit_results = []
for bin_i in range(0, nbins):
fr = plot_pull(pulls,
centre_of_mass,
channel,
variable,
k_value,
tau_value,
output_folder,
output_formats,
bin_index=bin_i,
n_bins=nbins)
fit_results.append(fr)
plot_fit_results(fit_results, centre_of_mass, channel, variable, k_value,
tau_value, output_folder, output_formats, bins)
# plot all bins
plot_pull(pulls, centre_of_mass, channel, variable, k_value, tau_value,
output_folder, output_formats)
del pulls # deleting to make space in memory
difference = get_data(files, subset='difference')
plot_difference(difference, centre_of_mass, channel, variable, k_value,
tau_value, output_folder, output_formats)
parser.add_option("-p", "--pathToBATOutputFiles", dest="pathToBATOutputFiles", default=None,
help="Specify the path where the BAT output files you want to move to hdfs are located.\
Format should be e.g /storage/<username>/<folder>/<cmssw_folder>/src/")
(options, _) = parser.parse_args()
if not options.pathToBATOutputFiles:
print "No path to files to be moved. Exiting."
sys.exit()
if options.centreOfMassEnergy != 7 and options.centreOfMassEnergy != 8:
print "Centre of mass energy must be 7 or 8 TeV"
print "You've chosen", options.centreOfMassEnergy
sys.exit()
#set up the config according to the centre of mass energy
config = XSectionConfig(options.centreOfMassEnergy)
#Get the luminosity for the centre of mass energy
luminosity = config.luminosities[options.centreOfMassEnergy]
#Get current working directory
current_working_directory = os.getcwd()
#Get folder to move files to
path_to_AN_folder = config.path_to_files
#move log files separately first, since there is no "logs" category in categories_and_prefixes
make_folder_if_not_exists(path_to_AN_folder + '/logs/')
command = 'mv ' + options.pathToBATOutputFiles + '/*' + str(options.centreOfMassEnergy) + 'TeV*.log ' + path_to_AN_folder + '/logs/'
if options.doNothing:
print "command = ", command
from config.summations_7TeV import sample_summations
from config import XSectionConfig
from tools.file_utilities import make_folder_if_not_exists
from tools.file_utilities import merge_ROOT_files
import os
import subprocess
import time
new_files = []
config_7TeV = XSectionConfig(7)
#Can not output to DICE, and since we are now working with input files on /hdfs, need to find somewhere to output the merged files
#this script creates
#So, create folders in working directory with structure: AN-XX-XXX_Xth_draft/XTeV/<central/BJet_up/Light_Jet_up/etc. for all categories>
#NOTE: YOU WILL THEN HAVE TO MOVE THESE MERGED FILES MANUALLY TO THE APPROPRIATE LOCATION IN /hdfs/TopQuarkGroup/results/histogramfiles/...
# first get current working directory
current_working_directory = os.getcwd()
path_to_AN_folder = config_7TeV.path_to_files
# change path from /hdfs to current working directory
path_to_AN_folder = path_to_AN_folder.replace("/hdfs/TopQuarkGroup/results/histogramfiles", current_working_directory)
#loop through all categories (e.g. central, BJet_up, LightJet_up, etc....) and make folder
for category in config_7TeV.categories_and_prefixes.keys():
make_folder_if_not_exists( path_to_AN_folder + "/" + category)
# merge generator systematics histogram files
for sample, input_samples in sample_summations.iteritems():
if not sample in ['WJets', 'VJets-matchingup',
'VJets-matchingdown', 'VJets-scaleup',
def plot(regularisation_settings, results, use_current_k_values=False):
variable = regularisation_settings.variable
channel = regularisation_settings.channel
com = regularisation_settings.centre_of_mass_energy
output_folder = regularisation_settings.output_folder
output_format = regularisation_settings.output_format
measurement_config = XSectionConfig(com)
name = 'reg_param_from_global_correlation_%s_channel_%s' % (channel,
variable)
hp = Histogram_properties()
hp.name = name
hp.x_axis_title = r'log($\tau$)'
hp.y_axis_title = r'$\bar{\rho}(\tau)$'
hp.title = r'global correlation for $%s$, %s channel, $\sqrt{s} = %d$ TeV'
hp.title = hp.title % (variables_latex[variable], channel, com)
k_results, tau_results = results
optimal_tau, minimal_rho, tau_values, rho_values = tau_results
optimal_k, optimal_k_rho, k_values, k_tau_values, k_rho_values = k_results
plt.figure(figsize=(16, 16), dpi=200, facecolor='white')
plt.plot(tau_values, rho_values)
plt.plot(k_tau_values, k_rho_values, 'ro')
plt.title(hp.title, CMS.title)
plt.xlabel(hp.x_axis_title, CMS.x_axis_title)
plt.ylabel(hp.y_axis_title, CMS.y_axis_title)
plt.tick_params(**CMS.axis_label_major)
plt.tick_params(**CMS.axis_label_minor)
ax = plt.axes()
current_k, closest_tau, _, _ = get_k_tau_set(measurement_config, channel,
variable, results)
current_k_rho = k_rho_values[k_values.index(current_k)]
# first best k
tau_index = k_values.index(optimal_k)
closest_tau_best_k = k_tau_values[tau_index]
ax.annotate(
r"$\tau = %.2g$" % optimal_tau,
xy=(optimal_tau, minimal_rho),
xycoords='data',
xytext=(optimal_tau * 0.9, minimal_rho * 1.15),
textcoords='data',
bbox=dict(boxstyle="round4", fc="w"),
arrowprops=dict(
arrowstyle="fancy,head_length=0.4,head_width=0.4,tail_width=0.4",
connectionstyle="arc3"),
size=40,
)
ax.annotate(
r"$\tau(k_b = %d) = %.2g$" % (optimal_k, closest_tau_best_k),
xy=(closest_tau_best_k, optimal_k_rho),
xycoords='data',
xytext=(closest_tau_best_k * 10, optimal_k_rho),
textcoords='data',
bbox=dict(boxstyle="round4", fc="w"),
arrowprops=dict(arrowstyle="<-", connectionstyle="arc3", lw=3),
size=40,
)
# then current k
if use_current_k_values:
ax.annotate(
r"$\tau(k_c = %d) = %.2g$" % (current_k, closest_tau),
xy=(closest_tau, current_k_rho),
xycoords='data',
xytext=(closest_tau, current_k_rho * 0.9),
textcoords='data',
bbox=dict(boxstyle="round4", fc="w"),
arrowprops=dict(arrowstyle="<-", connectionstyle="arc3", lw=3),
size=40,
)
plt.xscale('log')
make_folder_if_not_exists(output_folder)
for f in output_format:
plt.savefig(output_folder + '/' + hp.name + '.' + f)
def morph_systematic(central, systematic, to_be_morphed):
for i, _ in enumerate(central):
value_central = ufloat(central[i][0], central[i][1])
value_systematic = ufloat(systematic[i][0], systematic[i][1])
scale_factor = (1 - (value_central - value_systematic) / value_central)
to_be_morphed[i][0] *= scale_factor.nominal_value # change value
to_be_morphed[i][1] = sqrt(
pow(to_be_morphed[i][1] * scale_factor.nominal_value, 2) +
scale_factor.std_dev**2) # change error
return to_be_morphed
if __name__ == '__main__':
measurement_config = XSectionConfig(8)
generator_systematics = measurement_config.generator_systematics
categories_and_prefixes = measurement_config.categories_and_prefixes
met_type = 'patType1CorrectedPFMet'
measurement_type = 'unfolded'
signal = 'Higgs'
variables = bin_edges.keys()
variables = ['MET']
category = 'central'
output_file = root_open('test_' + measurement_type + '.root', 'recreate')
# TODO: mix in signal
variable = 'MET'
category = 'central'
output_file = root_open('test.root', 'recreate')
# TODO: mix in signal
hist_properties.y_limits = (1e-20, 1e20)
path = output_base + '/' + COMEnergy + 'TeV/' + channel + '/' + variable + '/' + fit_variable + '/'
if options.test:
path = output_base + '/test/'
measurements = {}
for test, histogram in histograms[sample].iteritems():
measurements[test.replace('_', ' ')] = histogram
compare_measurements(
{},
measurements,
show_measurement_errors=False,
histogram_properties=hist_properties,
save_folder=path,
save_as=['pdf'])
def is_test_only(COMEnergy, channel, variable, fit_variable):
never = COMEnergy == '8'
gonna = channel == 'electron'
give = variable == 'MET'
you = fit_variable == 'absolute_eta'
up = True
return never and gonna and give and you and up
if __name__ == '__main__':
config = XSectionConfig(8) # assume same setup for all
options = None
main()
