def get_merged_bin_resolution(res_file, var, low_bin, high_bin):
'''
Return mean value of resolutions in fine bins of the merged bin.
'''
bin_contents = []
f = File(res_file)
res_hist = f.Get('res_r_' + var).Clone()
# change scope from file to memory
res_hist.SetDirectory(0)
f.close()
low_bin_n = res_hist.GetXaxis().FindBin(low_bin)
high_bin_n = res_hist.GetXaxis().FindBin(high_bin)
for bin_i in range(low_bin_n, high_bin_n + 1):
bin_content = res_hist.GetBinContent(bin_i)
# resolution couldnt be reconstructed (High GeV with low stats)
# remove these from list of resolutions
if bin_content == 0: continue
bin_contents.append(bin_content)
# print(bin_contents)
res = np.mean(bin_contents)
return res
def isValid(self):
# file has to exists
if not os.path.exists(self.file):
input_log.debug('File does not exist: ' + self.file)
return False
if self.hist_name:
with File.open(self.file) as f:
if not f.__contains__(self.hist_name):
msg = 'File "{0}" does not contain histogram "{1}"'
input_log.debug(msg.format(self.file, self.hist_name))
return False
if self.tree_name:
with File.open(self.file) as f:
if not f.__contains__(self.tree_name):
msg = 'File "{0}" does not contain tree "{1}"'
input_log.debug(msg.format(self.file, self.tree_name))
return False
tree = f[self.tree_name]
branchToCheck = self.branch
if '[' in branchToCheck and ']' in branchToCheck:
branchToCheck = branchToCheck.split('[')[0]
elif 'abs(' in branchToCheck:
branchToCheck = branchToCheck.split('(')[-1].split(')')[0]
if not tree.has_branch(branchToCheck):
msg = 'Tree "{0}" does not contain branch "{1}"'
input_log.debug(msg.format(self.tree_name, branchToCheck))
return False
return True
def read_timespan(filename):
t_file = File(filename, 'read')
m = ref_time.match(t_file.get('m_utc_span').GetTitle())
week1, second1, week2, second2 = int(m.group(1)), int(m.group(2)), int(
m.group(3)), int(m.group(4))
t_file.close()
return (week1 * 604800 + second1, week2 * 604800 + second2)
def find_maintenance(filename):
aux_file = File(filename, 'read')
aux_tree = aux_file.get('t_hk_obox')
maintenance_start = False
maintenance_list = []
gps_time_list = []
ship_time_list = []
for entry in aux_tree:
if entry.obox_is_bad > 0: continue
if entry.obox_mode.encode('hex') == '04':
if not maintenance_start:
maintenance_start = True
gps_time_list.append(entry.abs_gps_week * 604800 +
entry.abs_gps_second)
ship_time_list.append(entry.abs_ship_second)
else:
if maintenance_start:
maintenance_start = False
maintenance_list.append(
((ship_time_list[0] + ship_time_list[-1]) / 2,
(gps_time_list[0] + gps_time_list[-1]) / 2))
gps_time_list = []
ship_time_list = []
return [(int(x[0]), "%d:%d" % (int(x[1] / 604800), int(x[1] % 604800)))
for x in maintenance_list]
def get_histogram_from_file( histogram_path, input_file ):
current_btag, found_btag = find_btag(histogram_path)
root_file = File( input_file )
get_histogram = root_file.Get
if not found_btag or not current_btag in sumations.b_tag_summations.keys():
root_histogram = get_histogram( histogram_path )
if not is_valid_histogram( root_histogram, histogram_path, input_file ):
return
else:
listOfExclusiveBins = sumations.b_tag_summations[current_btag]
exclhists = []
for excbin in listOfExclusiveBins:
hist = get_histogram( histogram_path.replace( current_btag, excbin ) )
if not is_valid_histogram( hist, histogram_path.replace( current_btag, excbin ), input_file ):
return
exclhists.append( hist )
root_histogram = exclhists[0].Clone()
for hist in exclhists[1:]:
root_histogram.Add( hist )
gcd()
histogram = None
# change from float to double
if root_histogram.TYPE == 'F':
histogram = root_histogram.empty_clone(type='D')
histogram.Add(root_histogram)
else:
histogram = root_histogram.Clone()
root_file.Close()
return histogram
def get_merged_bin_resolution(res_file, var, low_bin, high_bin):
'''
Return mean value of resolutions in fine bins of the merged bin.
'''
bin_contents = []
f = File( res_file )
res_hist = f.Get( 'res_r_'+var ).Clone()
# change scope from file to memory
res_hist.SetDirectory( 0 )
f.close()
low_bin_n = res_hist.GetXaxis().FindBin(low_bin)
high_bin_n = res_hist.GetXaxis().FindBin(high_bin)
for bin_i in range(low_bin_n, high_bin_n+1):
bin_content = res_hist.GetBinContent(bin_i)
# resolution couldnt be reconstructed (High GeV with low stats)
# remove these from list of resolutions
if bin_content == 0 : continue
bin_contents.append(bin_content)
# print(bin_contents)
res = np.mean(bin_contents)
return res
def find_orbitstart(filename):
LAT_LEN = 500
lat_deque = deque()
orbitstart_list = []
ppd_file = File(filename, 'read')
ppd_tree = ppd_file.get('t_ppd')
ready_flag = True
pre_diff = 0.0
cur_diff = 0.0
for entry in ppd_tree:
if entry.flag_of_pos != 0x55: continue
lat_deque.append(
(entry.latitude, entry.ship_time_sec, entry.utc_time_sec))
if len(lat_deque) < LAT_LEN:
pre_diff = lat_deque[-1][0] - lat_deque[0][0]
continue
else:
lat_deque.popleft()
cur_diff = lat_deque[-1][0] - lat_deque[0][0]
if ready_flag and pre_diff < 0 and cur_diff >= 0:
orbitstart_list.append(((lat_deque[-1][1] + lat_deque[0][1]) / 2,
(lat_deque[-1][2] + lat_deque[0][2]) / 2))
ready_flag = False
if not ready_flag and pre_diff > 0 and cur_diff <= 0:
ready_flag = True
pre_diff = cur_diff
return [(int(x[0]), "%d:%d" % (int(x[1] / 604800), int(x[1] % 604800)))
for x in orbitstart_list]
def addFileList(self, fileList):
if type(fileList) == type(list()):
for file in fileList:
self.files[file] = File(self.basepath + "/" + file + ".root",
"read")
if isinstance(fileList, dict):
import itertools
useList = list(
itertools.chain.from_iterable(list(fileList.values())))
for file in useList:
try:
self.files[file] = File(
self.basepath + "/" + file + ".root", "read")
except:
yesno = input("file %s is not there continue? [y/n]" %
(file))
if yesno != "y":
import sys
sys.exit(1)
fileList = {
key: value
for key, value in list(fileList.items())
if file not in value
}
self._joinList = fileList
self._getGenNumbers()
self._addToScaledView()
def write_to_root_files(parsed_hists, parsed_log_hists, output_filename):
start = time.time()
f = TFile(output_filename[0], "RECREATE")
for var in parsed_hists.keys():
index = 0
for mod_hist in parsed_hists[var]:
hist = copy.deepcopy(mod_hist.hist())
hist.SetName("{}#{}".format(var, index))
hist.Write()
index += 1
f.Close()
f = TFile(output_filename[1], "RECREATE")
for var in parsed_log_hists.keys():
index = 0
for mod_hist in parsed_log_hists[var]:
hist = copy.deepcopy(mod_hist.hist())
hist.SetName("{}#{}".format(var, index))
hist.Write()
index += 1
f.Close()
end = time.time()
def get_response_histogram(responseFileName, variable, channel):
'''
clones the response matrix from file
'''
responseFile = File(responseFileName, 'read')
folder = '{variable}_{channel}'.format(variable=variable, channel=channel)
h_response = responseFile.Get(folder).responseVis_without_fakes.Clone()
return asrootpy(h_response)
def setUp(self):
f = File('test.root', 'recreate')
tree = create_test_tree()
h = create_test_hist()
h.write()
tree.write()
f.write()
f.Close()
def get_electron_normalisation(met_bin, b_tag):
global electron_data_file
input_file = File(electron_data_file)
histogram_for_estimation = 'TTbarPlusMetAnalysis/EPlusJets/QCD e+jets PFRelIso/BinnedMETAnalysis/Electron_patType1CorrectedPFMet_bin_%s/electron_pfIsolation_03_%s' % (
met_bin, b_tag)
input_histogram = input_file.Get(histogram_for_estimation)
result = estimate_with_fit_to_relative_isolation(input_histogram)
value, error = result['value'], result['error']
return value, error
def get_histograms( variable, options ):
config = XSectionConfig( 13 )
path_electron = ''
path_muon = ''
path_combined = ''
histogram_name = ''
if options.visiblePhaseSpace:
histogram_name = 'responseVis_without_fakes'
else :
histogram_name = 'response_without_fakes'
if variable == 'HT':
path_electron = 'unfolding_HT_analyser_electron_channel/%s' % histogram_name
path_muon = 'unfolding_HT_analyser_muon_channel/%s' % histogram_name
path_combined = 'unfolding_HT_analyser_COMBINED_channel/%s' % histogram_name
else :
path_electron = 'unfolding_%s_analyser_electron_channel_patType1CorrectedPFMet/%s' % ( variable, histogram_name )
path_muon = 'unfolding_%s_analyser_muon_channel_patType1CorrectedPFMet/%s' % ( variable, histogram_name )
path_combined = 'unfolding_%s_analyser_COMBINED_channel_patType1CorrectedPFMet/%s' % ( variable, histogram_name )
histogram_information = [
{'file': config.unfolding_central_raw,
'CoM': 13,
'path':path_electron,
'channel':'electron'},
{'file':config.unfolding_central_raw,
'CoM': 13,
'path':path_muon,
'channel':'muon'},
]
if options.combined:
histogram_information = [
{'file': config.unfolding_central_raw,
'CoM': 13,
'path': path_combined,
'channel':'combined'},
]
for histogram in histogram_information:
f = File( histogram['file'] )
# scale to lumi
# nEvents = f.EventFilter.EventCounter.GetBinContent( 1 ) # number of processed events
# config = XSectionConfig( histogram['CoM'] )
# lumiweight = config.ttbar_xsection * config.new_luminosity / nEvents
lumiweight = 1
histogram['hist'] = f.Get( histogram['path'] ).Clone()
histogram['hist'].Scale( lumiweight )
# change scope from file to memory
histogram['hist'].SetDirectory( 0 )
f.close()
return histogram_information
def get_histograms(config, variable, args):
'''
Return a dictionary of the unfolding histogram informations (inc. hist)
'''
path_electron = ''
path_muon = ''
path_combined = ''
histogram_name = 'response_without_fakes'
if args.visiblePhaseSpace:
histogram_name = 'responseVis_without_fakes'
path_electron = '%s_electron/%s' % (variable, histogram_name)
path_muon = '%s_muon/%s' % (variable, histogram_name)
path_combined = '%s_combined/%s' % (variable, histogram_name)
histogram_information = [
{
'file': config.unfolding_central_raw,
'CoM': 13,
'path': path_electron,
'channel': 'electron'
},
{
'file': config.unfolding_central_raw,
'CoM': 13,
'path': path_muon,
'channel': 'muon'
},
]
if args.combined:
histogram_information = [
{
'file': config.unfolding_central_raw,
'CoM': 13,
'path': path_combined,
'channel': 'combined'
},
]
for histogram in histogram_information:
lumiweight = 1
f = File(histogram['file'])
histogram['hist'] = f.Get(histogram['path']).Clone()
# scale to current lumi
lumiweight = config.luminosity_scale
if round(lumiweight, 1) != 1.0:
print("Scaling to {}".format(lumiweight))
histogram['hist'].Scale(lumiweight)
# change scope from file to memory
histogram['hist'].SetDirectory(0)
f.close()
return histogram_information
def get_histograms( config, variable, args ):
'''
Return a dictionary of the unfolding histogram informations (inc. hist)
'''
path_electron = ''
path_muon = ''
path_combined = ''
histogram_name = 'response_without_fakes'
if args.visiblePhaseSpace:
histogram_name = 'responseVis_without_fakes'
path_electron = '%s_electron/%s' % ( variable, histogram_name )
path_muon = '%s_muon/%s' % ( variable, histogram_name )
path_combined = '%s_combined/%s' % ( variable, histogram_name )
histogram_information = [
{
'file' : config.unfolding_central_raw,
'CoM' : 13,
'path' : path_electron,
'channel' :'electron'
},
{
'file' : config.unfolding_central_raw,
'CoM' : 13,
'path' : path_muon,
'channel' :'muon'
},
]
if args.combined:
histogram_information = [
{
'file' : config.unfolding_central_raw,
'CoM' : 13,
'path' : path_combined,
'channel' : 'combined'
},
]
for histogram in histogram_information:
lumiweight = 1
f = File( histogram['file'] )
histogram['hist'] = f.Get( histogram['path'] ).Clone()
# scale to current lumi
lumiweight = config.luminosity_scale
if round(lumiweight, 1) != 1.0:
print( "Scaling to {}".format(lumiweight) )
histogram['hist'].Scale( lumiweight )
# change scope from file to memory
histogram['hist'].SetDirectory( 0 )
f.close()
return histogram_information
def get_histograms( variable ):
config_7TeV = XSectionConfig( 7 )
config_8TeV = XSectionConfig( 8 )
path_electron = ''
path_muon = ''
histogram_name = 'response_without_fakes'
if variable == 'MET':
path_electron = 'unfolding_MET_analyser_electron_channel_patType1CorrectedPFMet/%s' % histogram_name
path_muon = 'unfolding_MET_analyser_muon_channel_patType1CorrectedPFMet/%s' % histogram_name
elif variable == 'HT':
path_electron = 'unfolding_HT_analyser_electron_channel/%s' % histogram_name
path_muon = 'unfolding_HT_analyser_muon_channel/%s' % histogram_name
elif variable == 'ST':
path_electron = 'unfolding_ST_analyser_electron_channel_patType1CorrectedPFMet/%s' % histogram_name
path_muon = 'unfolding_ST_analyser_muon_channel_patType1CorrectedPFMet/%s' % histogram_name
elif variable == 'MT':
path_electron = 'unfolding_MT_analyser_electron_channel_patType1CorrectedPFMet/%s' % histogram_name
path_muon = 'unfolding_MT_analyser_muon_channel_patType1CorrectedPFMet/%s' % histogram_name
elif variable == 'WPT':
path_electron = 'unfolding_WPT_analyser_electron_channel_patType1CorrectedPFMet/%s' % histogram_name
path_muon = 'unfolding_WPT_analyser_muon_channel_patType1CorrectedPFMet/%s' % histogram_name
histogram_information = [
{'file': config_7TeV.unfolding_madgraph_raw,
'CoM': 7,
'path':path_electron,
'channel':'electron'},
{'file':config_7TeV.unfolding_madgraph_raw,
'CoM': 7,
'path':path_muon,
'channel':'muon'},
{'file':config_8TeV.unfolding_madgraph_raw,
'CoM': 8,
'path':path_electron,
'channel':'electron'},
{'file':config_8TeV.unfolding_madgraph_raw,
'CoM': 8,
'path':path_muon,
'channel':'muon'},
]
for histogram in histogram_information:
f = File( histogram['file'] )
# scale to lumi
nEvents = f.EventFilter.EventCounter.GetBinContent( 1 ) # number of processed events
config = XSectionConfig( histogram['CoM'] )
lumiweight = config.ttbar_xsection * config.new_luminosity / nEvents
histogram['hist'] = f.Get( histogram['path'] ).Clone()
histogram['hist'].Scale( lumiweight )
# change scope from file to memory
histogram['hist'].SetDirectory( 0 )
f.close()
return histogram_information
def can_open_ROOT_file(filename):
passesCheck = False
try:
openFile = File(filename, 'r')
if openFile:
passesCheck = True
openFile.Close()
except:
print "Could not open ROOT file"
return passesCheck
def calculate_resolutions(variable,
bin_edges=[],
channel='combined',
res_to_plot=False):
'''
Calculate the resolutions in the bins using the residual method
'''
f = File('unfolding/13TeV/unfolding_TTJets_13TeV.root')
fineBinEdges_path = '{}_{}/responseVis_without_fakes'.format(
variable, channel)
fineBinEdges_hist2d = f.Get(fineBinEdges_path).Clone()
fineBinEdges_hist1d = asrootpy(fineBinEdges_hist2d.ProjectionX())
fineBinEdges = list(fineBinEdges_hist1d.xedges())
nFineBins = len(fineBinEdges) - 1
tmp_residual_path = '{}_{}/residuals/Residuals_Bin_'.format(
variable, channel)
# Absolute lepton eta can have multiple fine bins at the same precision as wide bins. Only taking first.
if variable == 'abs_lepton_eta':
fineBinEdges = [round(entry, 2) for entry in fineBinEdges]
# For N Bin edges find resolutions of bins
resolutions = []
for i in range(len(bin_edges) - 1):
list_of_fine_bins = []
# Find fine bin edges in wide bins
for j, fine_bin_edge in enumerate(fineBinEdges):
if fine_bin_edge >= bin_edges[i] and fine_bin_edge < bin_edges[
i + 1] and j < nFineBins:
list_of_fine_bins.append(j + 1)
# Sum the residuals of the fine bins
for fine_bin in list_of_fine_bins:
if fine_bin == list_of_fine_bins[0]:
fineBin_histRes = f.Get(tmp_residual_path +
str(fine_bin)).Clone()
else:
fineBin_histRes_tmp = f.Get(tmp_residual_path +
str(fine_bin)).Clone()
fineBin_histRes.Add(fineBin_histRes, fineBin_histRes_tmp, 1.0,
1.0)
# Get the quantile at 68% = 1 sigma = Resolution
interval = np.array([0.])
quantile = np.array([0.68])
fineBin_histRes.GetQuantiles(1, interval, quantile)
resolutions.append(round(interval[0], 2))
if res_to_plot:
plotting_resolution(variable, channel, fineBin_histRes,
round(interval[0], 2), i, bin_edges[i],
bin_edges[i + 1])
return resolutions
def root_file_to_hist(input_filename, hist_templates):
hists = copy.deepcopy(hist_templates)
root_file = TFile(input_filename, "read")
for var in hists.keys():
mod_hist = hists[var]
hist = root_file.Get(var)
mod_hist.replace_hist(hist)
return hists
def getHistFromTree(self,
bins,
xmin,
xmax,
xtitle,
cut,
value,
tree,
weight=None):
from rootpy.plotting import Hist
import random, string, os
self.clearHists()
for f in self.files:
try:
_tree = self.files[f].Get(tree)
except AttributeError as e:
log_plotlib.warning("No %s in %s" % (tree, f))
log_plotlib.warning(
"Will try without %s, and add an empty hist." % f)
log_plotlib.warning(e)
self.hists[f] = Hist(binns, xmin, xmax)
continue
self.hists[f] = Hist(bins, xmin, xmax)
self.hists[f].GetXaxis().SetTitle(xtitle)
try:
if weight is None:
_tree.Draw(value, selection=cut, hist=self.hists[f])
else:
#_tree.Draw(value,selection="(%s)*(%s)"%(cut,weight),hist=self.hists[f])
tmpFileName = ''.join(
random.choice(string.ascii_lowercase)
for i in range(4))
tmpFile = File("/tmp/%s.root" % tmpFileName, "recreate")
#sel_tree=_tree.copy_tree(selection=cut)
sel_tree = asrootpy(_tree.CopyTree(cut))
##print weight
sel_tree.Draw(value, selection=weight, hist=self.hists[f])
tmpFile.Close()
os.remove("/tmp/%s.root" % tmpFileName)
except Exception as e:
log_plotlib.info("error:%s" % (e))
log_plotlib.info("file :%s" % (f))
log_plotlib.info("Perhaps try this one:")
for i in _tree.glob("*"):
log_plotlib.info(i)
raise RuntimeError("Will stop here!")
self.hists[f].Scale(self._getWeight(f))
self.hists[f].Sumw2()
if self._joinList is not False:
self.joinList(self._joinList)
for hist in self.hists:
if hist in self.style:
self.hists[hist].decorate(**self.style[hist])
def open_file(self, filename, begin,
end): # cut data by utc time, utc_week:utc_second
self.t_file_name = basename(filename)
self.t_file_in = File(filename, 'read')
self.t_tree_ppd = self.t_file_in.get('t_ppd')
self.t_tree_ppd.create_buffer()
self.utc_time_span = self.t_file_in.get('m_utc_span').GetTitle()
m = re.compile(
r'(\d+):(\d+)\[\d+\] => (\d+):(\d+)\[\d+\]; \d+/\d+').match(
self.utc_time_span)
self.first_utc_time_sec = float(m.group(1)) * 604800 + float(
m.group(2))
self.last_utc_time_sec = float(m.group(3)) * 604800 + float(m.group(4))
if begin != 'begin':
m = re.compile(r'(\d+):(\d+)').match(begin)
self.begin_utc_time_sec = float(m.group(1)) * 604800 + float(
m.group(2))
if self.begin_utc_time_sec - self.first_utc_time_sec < _MIN_DIFF:
print 'WARNING: begin utc time is out of range: ' + str(
self.begin_utc_time_sec - self.first_utc_time_sec)
return False
else:
self.begin_utc_time_sec = -1
if end != 'end':
m = re.compile(r'(\d+):(\d+)').match(end)
self.end_utc_time_sec = float(m.group(1)) * 604800 + float(
m.group(2))
if self.last_utc_time_sec - self.end_utc_time_sec < _MIN_DIFF:
print 'WARNING: end utc time is out of range: ' + str(
self.last_utc_time_sec - self.end_utc_time_sec)
return False
else:
self.end_utc_time_sec = -1
if self.begin_utc_time_sec > 0 and self.end_utc_time_sec > 0 and self.end_utc_time_sec - self.begin_utc_time_sec < _MIN_DIFF:
print 'WARNING: time span between begin and end utc time is too small: ' + str(
self.end_utc_time_sec - self.begin_utc_time_sec)
return False
if self.begin_utc_time_sec > 0:
self.begin_entry = self.__find_entry(self.begin_utc_time_sec)
if self.begin_entry < 0:
print "WARNING: cannot find begin entry."
return False
else:
self.begin_entry = 0
if self.end_utc_time_sec > 0:
self.end_entry = self.__find_entry(self.end_utc_time_sec)
if self.end_entry < 0:
print "WARNING: cannot find end entry."
return False
else:
self.end_entry = self.t_tree_ppd.get_entries()
return True
def read_timespan(filename, dat_type):
t_file = File(filename, 'read')
m = ref_time.match(t_file.get(tnamed_dict[dat_type]).GetTitle())
week1, second1, week2, second2 = int(m.group(1)), int(m.group(2)), int(
m.group(3)), int(m.group(4))
t_file.close()
time_seconds_begin = week1 * 604800 + second1
time_seconds_end = week2 * 604800 + second2
beijing_time_begin = datetime(1980, 1, 6, 0, 0, 0) + timedelta(
seconds=time_seconds_begin - leap_seconds_dict[dat_type] + 28800)
beijing_time_end = datetime(1980, 1, 6, 0, 0, 0) + timedelta(
seconds=time_seconds_end - leap_seconds_dict[dat_type] + 28800)
return (beijing_time_begin, beijing_time_end)
def getStystPlot(hist):
import os
if os.path.exists("syst/"+hist.replace("/","")+"_syst.root"):
systFile=File("syst/"+hist.replace("/","")+"_syst.root","read")
stysthist=systFile.Get(hist)
stysthist.SetDirectory(0)
stysthist.SetLineWidth(0)
stysthist.SetLineColor(0)
else:
raise IOError("no syst for %s"%(hist))
if "Phi" in hist:
stysthist.Smooth()
return stysthist
def convert_unfolding_histograms(file_name,
histograms_to_load=['truth',
'fake', 'measured', 'response',
'response_withoutFakes', 'response_without_fakes',
'EventCounter',
]):
file_start = Timer()
print 'Converting', file_name
histograms = {}
with File(file_name) as f:
for path, _, objects in f.walk():
# keep only unfolding and EventFilter
if path.startswith('unfolding_') or path == 'EventFilter':
histograms[path] = {}
for hist_name in objects:
if hist_name in histograms_to_load:
hist = f.Get(path + '/' + hist_name).Clone()
hist.SetDirectory(0)
histograms[path][hist_name] = hist
new_histograms = {}
# rebin
for path, hists in histograms.iteritems():
new_histograms[path] = {}
variable = ''
if not path == 'EventFilter':
variable = path.split('_')[1]
for name, hist in hists.iteritems():
if name == 'EventCounter':
new_histograms[path][name] = hist.Clone()
else:
new_hist = hist.rebinned(bin_edges_vis[variable])
if 'TH2' in new_hist.class_name():
new_hist = new_hist.rebinned(bin_edges_vis[variable], axis=1)
new_histograms[path][name] = new_hist
# save_to_file
output = File(file_name.replace('.root', '_asymmetric.root'), 'recreate')
for path, hists in new_histograms.iteritems():
directory = output.mkdir(path)
directory.cd()
for name, hist in hists.iteritems():
if name == 'response_withoutFakes': # fix this name
hist.Write('response_without_fakes')
else:
hist.Write(name)
output.close()
secs = file_start.elapsed_time()
print 'File %s converted in %d seconds' % (file_name, secs)
def parse_to_root_file(input_filename, output_filename, hist_templates):
print "Parsing {} to {}".format(input_filename, output_filename)
parsed_hists = parse_file(input_filename, copy.deepcopy(hist_templates))
f = TFile(output_filename, "RECREATE")
for var in parsed_hists.keys():
mod_hist = parsed_hists[var]
hist = copy.deepcopy(mod_hist.hist())
hist.SetName("{}".format(var))
hist.Write()
f.Close()
def write_to_root_file(output_filename, parsed_hists):
f = TFile(output_filename, "RECREATE")
for var in parsed_hists.keys():
index = 0
for mod_hist in parsed_hists[var]:
hist = copy.deepcopy(mod_hist.hist())
hist.SetName("{}#{}".format(var, index))
hist.Write()
index += 1
f.Close()
def setUp(self):
create_test_tree('test.root')
# append a histogram
with File.open('test.root', 'a+') as f:
h = create_test_hist()
h.write()
f.write()
def addAllFiles(self, tag="", veto=None, regexpr=None, joinName=None):
if self.basepath == None:
raise RuntimeError(
"You must set a basepath to add all files from one directory!")
if regexpr is not None:
import re, os
fileList = [
f for f in os.listdir(self.basepath + "/")
if re.search(r'%s' % (regexpr), f)
]
else:
import glob
fileList = glob.glob(self.basepath + "/*" + tag + "*.root")
tmpList = []
for file in fileList:
if veto is not None:
vetoed = False
for v in veto:
if v.lower() in file.split("/")[-1].lower():
vetoed = True
if vetoed:
log_plotlib.info("vetoed file: %s" % file)
continue
name = file.split("/")[-1].replace(".root", "")
self.files[name] = File(file, "read")
tmpList.append(name)
self._getGenNumbers()
self._addToScaledView()
if joinName is not None:
if self._joinList is not False:
self._joinList[joinName] = tmpList
else:
self._joinList = OrderedDict()
self._joinList[joinName] = tmpList
def cleanFile(filename):
testfile = File(filename, 'update')
nHistograms = 0
print "Deleting inclusive bin histograms histograms"
for folder, emptyThing, histograms in testfile:
for histogram in histograms:
currentPath = folder + '/' + histogram
for btag in btag_bins_inclusive:
if btag in histogram:
nHistograms += 1
# print 'Deleting:', currentPath
testfile.Delete(currentPath + ';*')
print 'Deleted', nHistograms, 'histograms'
print 'Closing file', filename
testfile.Close()
print 'Closed file'
def setUp(self):
create_test_tree('test.root')
# append a histogram
with File.open ('test.root', 'a+') as f:
h = create_test_hist()
h.write()
f.write()
def main():
config = XSectionConfig(13)
# method = 'RooUnfoldSvd'
method = 'RooUnfoldBayes'
file_for_unfolding = File(config.unfolding_central, 'read')
for channel in ['electron', 'muon', 'combined']:
for variable in bin_edges.keys():
tau_value = get_tau_value(config, channel, variable)
h_truth, h_measured, h_response, h_fakes = get_unfold_histogram_tuple(
inputfile=file_for_unfolding,
variable=variable,
channel=channel,
met_type=config.met_type,
centre_of_mass=config.centre_of_mass_energy,
ttbar_xsection=config.ttbar_xsection,
luminosity=config.luminosity,
load_fakes=False,
visiblePS=False,
)
unfolding = Unfolding(h_truth,
h_measured,
h_response,
h_fakes,
method=method,
k_value=-1,
tau=tau_value)
unfolded_data = unfolding.closureTest()
plot_closure(h_truth, unfolded_data, variable, channel,
config.centre_of_mass_energy, method)
def get_muon_normalisation(met_bin, b_tag):
global path_to_files
muon_qcd_file = path_to_files + 'central/QCD_Pt-20_MuEnrichedPt-15_5050pb_PFElectron_PFMuon_PF2PATJets_PFMET.root'
input_file = File(muon_qcd_file)
histogram_for_estimation = 'TTbarPlusMetAnalysis/MuPlusJets/Ref selection/BinnedMETAnalysis/Muon_patType1CorrectedPFMet_bin_%s/muon_AbsEta_%s' % (
met_bin, b_tag)
#if not correctly scaled, rescale here
#
input_histogram = input_file.Get(histogram_for_estimation)
scale_factor = 1.21
value = input_histogram.Integral() * scale_factor
error = sum([
input_histogram.GetBinError(bin_i) * scale_factor
for bin_i in range(1, input_histogram.nbins())
])
return value, error
def get_histograms_from_trees(
trees=[],
branch='var',
weightBranch='EventWeight',
selection='1',
files={},
verbose=False,
nBins=40,
xMin=0,
xMax=100,
ignoreUnderflow=True,
):
histograms = {}
nHistograms = 0
# Setup selection and weight string for ttree draw
weightAndSelection = '( %s ) * ( %s )' % (weightBranch, selection)
for sample, input_file in files.iteritems():
root_file = File(input_file)
get_tree = root_file.Get
histograms[sample] = {}
for tree in trees:
tempTree = tree
if 'data' in sample and ('Up' in tempTree or 'Down' in tempTree):
tempTree = tempTree.replace('_' + tempTree.split('_')[-1], '')
currentTree = get_tree(tempTree)
root_histogram = Hist(nBins, xMin, xMax)
currentTree.Draw(branch, weightAndSelection, hist=root_histogram)
if not is_valid_histogram(root_histogram, tree, input_file):
return
# When a tree is filled with a dummy variable, it will end up in the underflow, so ignore it
if ignoreUnderflow:
root_histogram.SetBinContent(0, 0)
root_histogram.SetBinError(0, 0)
gcd()
nHistograms += 1
histograms[sample][tree] = root_histogram.Clone()
root_file.Close()
return histograms
def checkOnMC(unfolding, method):
global bins, nbins
RooUnfold.SVD_n_toy = 1000
pulls = []
for sub in range(1,9):
inputFile2 = File('../data/unfolding_merged_sub%d.root' % sub, 'read')
h_data = asrootpy(inputFile2.unfoldingAnalyserElectronChannel.measured.Rebin(nbins, 'measured', bins))
nEvents = inputFile2.EventFilter.EventCounter.GetBinContent(1)
lumiweight = 164.5 * 5050 / nEvents
# print sub, nEvents
h_data.Scale(lumiweight)
doUnfoldingSequence(unfolding, h_data, method, '_sub%d' %sub)
pull = unfolding.pull_inputErrorOnly()
# unfolding.printTable()
pulls.append(pull)
unfolding.Reset()
allpulls = []
for pull in pulls:
allpulls.extend(pull)
h_allpulls = Hist(100,-30,30)
filling = h_allpulls.Fill
for entry in allpulls:
filling(entry)
fit = h_allpulls.Fit('gaus', 'WWS')
h_fit = asrootpy(h_allpulls.GetFunction("gaus").GetHistogram())
canvas = Canvas(width=1600, height=1000)
canvas.SetLeftMargin(0.15)
canvas.SetBottomMargin(0.15)
canvas.SetTopMargin(0.10)
canvas.SetRightMargin(0.05)
h_allpulls.Draw()
fit.Draw('same')
canvas.SaveAs('plots/Pull_allBins_withFit.png')
plt.figure(figsize=(16, 10), dpi=100)
rplt.errorbar(h_allpulls, label=r'Pull distribution for all bins', emptybins=False)
rplt.hist(h_fit, label=r'fit')
plt.xlabel('(unfolded-true)/error', CMS.x_axis_title)
plt.ylabel('entries', CMS.y_axis_title)
plt.title('Pull distribution for all bins', CMS.title)
plt.tick_params(**CMS.axis_label_major)
plt.tick_params(**CMS.axis_label_minor)
plt.legend(numpoints=1)
plt.savefig('plots/Pull_allBins.png')
#individual bins
for bin_i in range(nbins):
h_pull = Hist(100,-30,30)
for pull in pulls:
h_pull.Fill(pull[bin_i])
plt.figure(figsize=(16, 10), dpi=100)
rplt.errorbar(h_pull, label=r'Pull distribution for bin %d' % (bin_i + 1), emptybins=False)
plt.xlabel('(unfolded-true)/error', CMS.x_axis_title)
plt.ylabel('entries', CMS.y_axis_title)
plt.title('Pull distribution for bin %d' % (bin_i + 1), CMS.title)
plt.savefig('Pull_bin_%d.png' % (bin_i + 1))
def test_file_open():
fname = 'test_file_open.root'
with File.open(fname, 'recreate'):
pass
with root_open(fname):
pass
os.unlink(fname)
def test_file_open():
fname = 'test_file_open.root'
with File.open(fname, 'recreate'):
pass
with root_open(fname):
pass
os.unlink(fname)
def write_histogram(var, hname, weight, samples, sn, sampn, cuts, cuts_antiiso, outdir, channel, coupling, binning=None, plot_range=None, asymmetry=None, mtmetcut=None):
weight_str = weight
samp = samples[sampn]
outfile = File(outdir + "/%s_%s.root" % (sampn,hname), "RECREATE")
if sn=="DATA":
weight_str = "1"
if var == "eta_lj":
var = "abs("+var+")"
qcd_extra = None
#This is a really ugly way of adding the QCD shape variation, but works. Restructure the whole thing in the future
if "iso__down" in hname:
if var == "abs(eta_lj)":
cuts_antiiso = str(Cuts.eta_fit_antiiso_down(channel))
qcd_extra = str(Cuts.eta_fit_antiiso(channel)) #hack for now
else:
for x in str(cuts_antiiso).split("&&"):
if "mva" in x:
cut = x
cut = cut.replace("(","").replace(")","")
cuts_antiiso = str(Cuts.mva_antiiso_down(channel, mva_var=var)) + " && ("+cut+")"
qcd_extra = str(Cuts.mva_antiiso(channel, mva_var=var)) + " && ("+cut+")" #hack for now
elif "iso__up" in hname:
if var == "abs(eta_lj)":
cuts_antiiso = str(Cuts.eta_fit_antiiso_up(channel))
qcd_extra = str(Cuts.eta_fit_antiiso(channel)) #hack for now
else:
for x in str(cuts_antiiso).split("&&"):
if "mva" in x:
cut = x
cut = cut.replace("(","").replace(")","")
cuts_antiiso = str(Cuts.mva_antiiso_up(channel, mva_var=var)) + " && ("+cut+")"
qcd_extra = str(Cuts.mva_antiiso(channel, mva_var=var)) + " && ("+cut+")" #hack for now
hist = create_histogram_for_fit(sn, samp, weight_str, cuts, cuts_antiiso, channel, coupling, var, binning=binning, plot_range=plot_range, asymmetry=asymmetry, qcd_extra=qcd_extra, mtmetcut=mtmetcut)
outfile.cd() #Must cd after histogram creation
#Write histogram to file
logging.info("Writing histogram %s to file %s" % (hist.GetName(), outfile.GetPath()))
logging.info("%i entries, %.2f events" % (hist.GetEntries(), hist.Integral()))
hist.SetName(hname)
hist.SetDirectory(outfile)
#hist.Write() #Double write?
outfile.Write()
outfile.Close()
def get_histograms(variable, options):
config = XSectionConfig(13)
path_electron = ""
path_muon = ""
path_combined = ""
histogram_name = ""
if options.visiblePhaseSpace:
histogram_name = "responseVis_without_fakes"
else:
histogram_name = "response_without_fakes"
path_electron = "%s_electron/%s" % (variable, histogram_name)
path_muon = "%s_muon/%s" % (variable, histogram_name)
path_combined = "%s_combined/%s" % (variable, histogram_name)
histogram_information = [
{"file": config.unfolding_central_raw, "CoM": 13, "path": path_electron, "channel": "electron"},
{"file": config.unfolding_central_raw, "CoM": 13, "path": path_muon, "channel": "muon"},
]
if options.combined:
histogram_information = [
{"file": config.unfolding_central_raw, "CoM": 13, "path": path_combined, "channel": "combined"}
]
for histogram in histogram_information:
f = File(histogram["file"])
# scale to lumi
# nEvents = f.EventFilter.EventCounter.GetBinContent( 1 ) # number of processed events
# config = XSectionConfig( histogram['CoM'] )
# lumiweight = config.ttbar_xsection * config.new_luminosity / nEvents
lumiweight = 1
histogram["hist"] = f.Get(histogram["path"]).Clone()
histogram["hist"].Scale(lumiweight)
# change scope from file to memory
histogram["hist"].SetDirectory(0)
f.close()
return histogram_information
def load_theta_format(inf, styles=None):
from plots.common.sample_style import Styling
if not styles:
styles = {}
hists = SystematicHistCollection()
fi = File(inf)
ROOT.gROOT.cd()
#rate_sfs = load_fit_results("final_fit/results/%s.txt" % fit_results[channel])
for root, dirs, items in fi.walk():
for it in items:
hn = hname_decode(it)
variable, sample, syst, systdir = hn["var"], hn["sample"], hn["type"], hn["dir"]
k = os.path.join(root, it)
try:
it = fi.Get(k).Clone()
except:
raise Exception("Could not copy object %s" % k)
#Skip anything not a histogram
if not isinstance(it, ROOT.TH1F):
continue
if sample == "DATA":
sample = sample.lower()
if sample.lower() != "data":
sty = styles.get(sample.lower(), sample)
try:
Styling.mc_style(it, sty)
except:
logger.debug("Could not style histogram from sample: %s" % sample)
else:
Styling.data_style(it)
hists[variable][syst][systdir][sample]= it
#hists = hists.as_dict()
fi.Close()
return hists
def unfold_results( results, category, channel, k_value, h_truth, h_measured, h_response, h_fakes, method ):
global variable, path_to_JSON, options
h_data = value_error_tuplelist_to_hist( results, bin_edges[variable] )
unfolding = Unfolding( h_truth, h_measured, h_response, h_fakes, method = method, k_value = k_value )
# turning off the unfolding errors for systematic samples
if not category == 'central':
unfoldCfg.Hreco = 0
else:
unfoldCfg.Hreco = options.Hreco
h_unfolded_data = unfolding.unfold( h_data )
if options.write_unfolding_objects:
# export the D and SV distributions
SVD_path = path_to_JSON + '/unfolding_objects/' + channel + '/kv_' + str( k_value ) + '/'
make_folder_if_not_exists( SVD_path )
if method == 'TSVDUnfold':
SVDdist = File( SVD_path + method + '_SVDdistributions_' + category + '.root', 'recreate' )
directory = SVDdist.mkdir( 'SVDdist' )
directory.cd()
unfolding.unfoldObject.GetD().Write()
unfolding.unfoldObject.GetSV().Write()
# unfolding.unfoldObject.GetUnfoldCovMatrix(data_covariance_matrix(h_data), unfoldCfg.SVD_n_toy).Write()
SVDdist.Close()
else:
SVDdist = File( SVD_path + method + '_SVDdistributions_Hreco' + str( unfoldCfg.Hreco ) + '_' + category + '.root', 'recreate' )
directory = SVDdist.mkdir( 'SVDdist' )
directory.cd()
unfolding.unfoldObject.Impl().GetD().Write()
unfolding.unfoldObject.Impl().GetSV().Write()
h_truth.Write()
h_measured.Write()
h_response.Write()
# unfolding.unfoldObject.Impl().GetUnfoldCovMatrix(data_covariance_matrix(h_data), unfoldCfg.SVD_n_toy).Write()
SVDdist.Close()
# export the whole unfolding object if it doesn't exist
if method == 'TSVDUnfold':
unfolding_object_file_name = SVD_path + method + '_unfoldingObject_' + category + '.root'
else:
unfolding_object_file_name = SVD_path + method + '_unfoldingObject_Hreco' + str( unfoldCfg.Hreco ) + '_' + category + '.root'
if not os.path.isfile( unfolding_object_file_name ):
unfoldingObjectFile = File( unfolding_object_file_name, 'recreate' )
directory = unfoldingObjectFile.mkdir( 'unfoldingObject' )
directory.cd()
if method == 'TSVDUnfold':
unfolding.unfoldObject.Write()
else:
unfolding.unfoldObject.Impl().Write()
unfoldingObjectFile.Close()
del unfolding
return hist_to_value_error_tuplelist( h_unfolded_data )
def create_test_tree(filename='test.root'):
with File.open (filename, 'recreate') as f:
tree = Tree("test")
tree.create_branches(
{'x': 'F',
'y': 'F',
'z': 'F',
'i': 'I',
'EventWeight': "F"})
for i in xrange(10000):
tree.x = gauss(.5, 1.)
tree.y = gauss(.3, 2.)
tree.z = gauss(13., 42.)
tree.i = i
tree.EventWeight = 1.
tree.fill()
f.write()
def get_histogram_from_tree(**kwargs):
branch = kwargs.pop('branch')
weight_branches = kwargs.pop('weight_branches')
weight_branch = ' * '.join(weight_branches)
rul.debug('Weight branch expression: "{0}"'.format(weight_branch))
branches = [branch]
branches.extend(weight_branches)
selection_branches = []
if kwargs.has_key('selection_branches'):
selection_branches = kwargs.pop('selection_branches')
branches.extend(selection_branches)
input_file = kwargs.pop('input_file')
tree = kwargs.pop('tree')
selection = kwargs.pop('selection')
weight_and_selection = '( {0} ) * ( {1} )'.format(weight_branch, selection)
if kwargs.has_key('n_bins'):
hist = Hist(kwargs['n_bins'], kwargs['x_min'], kwargs['x_max'],
type = 'D')
if kwargs.has_key('bin_edges'):
hist = Hist(kwargs['bin_edges'], type = 'D')
with File.open(input_file) as f:
t = f[tree]
branchesToActivate = []
for b in branches:
if '[' in b and ']' in b:
branchesToActivate.append( b.split('[')[0] )
elif 'abs(' in b:
branchesToActivate.append( b.split('(')[-1].split(')')[0] )
elif '*' in b:
branchesToAdd = b.split('*')
for branch in branchesToAdd:
branchesToActivate.append( branch.replace('(','').replace(')','').replace(' ',''))
else :
branchesToActivate.append(b)
rul.debug('Activating branches: {0}'.format(branchesToActivate))
t.activate(branchesToActivate, exclusive = True)
t.Draw(branch, selection = weight_and_selection, hist = hist)
return hist
def main(options, args):
config = XSectionConfig(options.CoM)
variables = ['MET', 'HT', 'ST', 'WPT']
channels = ['electron', 'muon', 'combined']
m_file = 'normalised_xsection_patType1CorrectedPFMet.txt'
m_with_errors_file = 'normalised_xsection_patType1CorrectedPFMet_with_errors.txt'
path_template = args[0]
output_file = 'measurement_{0}TeV.root'.format(options.CoM)
f = File(output_file, 'recreate')
for channel in channels:
d = f.mkdir(channel)
d.cd()
for variable in variables:
dv = d.mkdir(variable)
dv.cd()
if channel == 'combined':
path = path_template.format(variable=variable,
channel=channel,
centre_of_mass_energy=options.CoM)
else:
kv = channel + \
'/kv{0}/'.format(config.k_values[channel][variable])
path = path_template.format(variable=variable,
channel=kv,
centre_of_mass_energy=options.CoM)
m = read_data_from_JSON(path + '/' + m_file)
m_with_errors = read_data_from_JSON(
path + '/' + m_with_errors_file)
for name, result in m.items():
h = make_histogram(result, bin_edges_full[variable])
h.SetName(name)
h.write()
for name, result in m_with_errors.items():
if not 'TTJet' in name:
continue
h = make_histogram(result, bin_edges_full[variable])
h.SetName(name + '_with_syst')
h.write()
dv.write()
d.cd()
d.write()
f.write()
f.close()
def get_histogram_from_tree(**kwargs):
branch = kwargs.pop("branch")
weight_branches = kwargs.pop("weight_branches")
weight_branch = " * ".join(weight_branches)
rul.debug('Weight branch expression: "{0}"'.format(weight_branch))
branches = [branch]
branches.extend(weight_branches)
selection_branches = []
if kwargs.has_key("selection_branches"):
selection_branches = kwargs.pop("selection_branches")
branches.extend(selection_branches)
input_file = kwargs.pop("input_file")
tree = kwargs.pop("tree")
selection = kwargs.pop("selection")
weight_and_selection = "( {0} ) * ( {1} )".format(weight_branch, selection)
if kwargs.has_key("n_bins"):
hist = Hist(kwargs["n_bins"], kwargs["x_min"], kwargs["x_max"], type="D")
if kwargs.has_key("bin_edges"):
hist = Hist(kwargs["bin_edges"], type="D")
with File.open(input_file) as f:
t = f[tree]
branchesToActivate = []
for b in branches:
if "[" in b and "]" in b:
branchesToActivate.append(b.split("[")[0])
elif "abs(" in b:
branchesToActivate.append(b.split("(")[-1].split(")")[0])
elif "*" in b:
branchesToAdd = b.split("*")
for branch in branchesToAdd:
branchesToActivate.append(branch.replace("(", "").replace(")", "").replace(" ", ""))
else:
branchesToActivate.append(b)
rul.debug("Activating branches: {0}".format(branchesToActivate))
t.activate(branchesToActivate, exclusive=True)
t.Draw(branch, selection=weight_and_selection, hist=hist)
return hist
bins = array('d', [0, 25, 45, 70, 100, 1000])
nbins = len(bins) - 1
parser = OptionParser()
parser.add_option("-n", "--n_toy_mc",
dest="n_toy_mc", default=100,
help="number of toy MC to create")
parser.add_option("-i", "--input",
dest="input_file", default='/storage/TopQuarkGroup/unfolding/unfolding_merged_sub1.root',
help="input file for templates")
parser.add_option("-o", "--output",
dest="output_file", default='../data/unfolding_toy_mc.root',
help="output file for toy MC")
(options, args) = parser.parse_args()
# define output file
output = File(options.output_file, 'recreate')
for channel in ['electron', 'muon']:
# get histograms
h_truth, h_measured, h_fakes, h_response_AsymBins, h_reco_truth, h_truth_selected = read_and_scale_histograms(channel)
directory = output.mkdir(channel)
directory.cd()
mkdir = directory.mkdir
cd = directory.cd
# generate toy MC
for i in range(1, options.n_toy_mc + 1):
mkdir('toy_%d' % i)
cd('toy_%d' % i) # should the numbering be transferred to the histograms?
if i % 100 == 0:
print 'Done %d toy MC' % i
new_histograms = get_new_set_of_histograms(h_truth, h_measured, h_fakes, h_response_AsymBins, h_reco_truth, h_truth_selected)
for hist in new_histograms:
def write_histos_to_file(hists, outdir, syst=""):
filename = outdir+"/"+generate_file_name(False, syst)
if len(syst)>0:
filename = filename.replace("/lqeta","")
#filename += ".root"
outfile = File(filename, "recreate")
#rint "writing to file", filename
for category in hists:
factor = 1.0
total_hist=hists[category][0].Clone()
total_hist.Reset("ICE")
#total_hist.Sumw2()
total_hist.SetNameTitle(category,category)
outfile.cd()
#print "CAT",category
for bin in range(1, total_hist.GetNbinsX()+1):
zero_error = 0.
zero_integral = 0.
nonzero_error = 0.
bin_sum = 0.
max_zero_error = 0.
zero_errors = {}
zero_errors[category] = []
for hist in hists[category]:
#print "hist", hist.GetBinContent(bin), hist.GetBinError(bin)**2
zero_errors[category].append(sys.float_info.max)
if hist.Integral()>0:
#factor = hist.Integral()/(math.sqrt(hist.GetEntries()) * total_hist.GetNbinsX())
#print "fact", factor,hist.Integral(), hist.GetEntries()
for bin1 in range(1, hist.GetNbinsX()+1):
if hist.GetBinContent(bin1) > 0 and hist.GetBinError(bin1) < zero_errors[category][-1]:
zero_errors[category][-1] = hist.GetBinError(bin1)**2
#print "error", min_nonzero_error
else:
zero_errors[category][-1] = 0.
if zero_errors[category][-1] > 10000:
#rint "ZERO error NOT ASSIGNED"
#or bin1 in range(1, hist.GetNbinsX()+1):
# print bin1, hist.GetBinContent(bin1), hist.GetBinError(bin1)
zero_errors[category][-1] = 0.
if hist.GetBinContent(bin) < 0.00001:
#zero_error += factor**2 * hist.Integral()
#zero_error += min_nonzero_error**2
zero_integral += hist.Integral()
else:
bin_sum += hist.GetBinContent(bin)
nonzero_error += hist.GetBinError(bin)**2
zero_errors[category][-1] = 0.
#for bin1 in range(1, hist.GetNbinsX()+1):
# print hist.GetBinContent(bin1), math.sqrt(hist.GetBinError(bin1))
#rint bin, hist.GetName(), hist.GetBinContent(bin), hist.GetBinError(bin), zero_errors[category][-1]
#print "...hist", hist.GetBinContent(bin), hist.GetBinError(bin)**2
total_hist.SetBinContent(bin, bin_sum)
zero_error = 0.
for err in zero_errors[category]:
if err > zero_error:
zero_error = err
#rint "ZERO error:", bin, math.sqrt(zero_error)
total_error = math.sqrt(nonzero_error + zero_error)
#rint math.sqrt(nonzero_error), math.sqrt(zero_error), total_error
total_hist.SetBinError(bin, total_error)
#print category, "bin", bin, "content", bin_sum, "error", total_error
#print category, "bin", bin, "weight", total_error**2/bin_sum
total_hist.Write()
"""for category in hists: #Imitate hadd
#factor = 1.0
total_hist=hists[category][0].Clone()
total_hist.Reset("ICE")
print "cat", category
total_hist.SetNameTitle(category,category)
total_hist.Sumw2()
outfile.cd()
for hist in hists[category]:
print "hist", hist, hist.GetName()
factor = 1.0
if hist.GetEntries()>0:
factor = hist.Integral()/hist.GetEntries()
for bin in range(1, total_hist.GetNbinsX()+1):
if hist.GetBinError(bin) < factor:
hist.SetBinError(bin, factor)
#total_hist.SetBinContent(bin, total_hist.GetBinContent(bin) + hist.GetBinContent(bin))
#total_hist.SetBinError(bin, total_hist.GetBinError(bin) + hist.GetBinError(bin))
total_hist.Add(hist)
total_hist.Write()"""
outfile.Write()
outfile.Close()
#!/usr/bin/env python
import sys
from os.path import basename
from rootpy.io import File
from rootpy.tree import Tree
if len(sys.argv) < 3:
print 'Usage: ' + basename(sys.argv[0]) + ' <decoded_data.root> <rate_file.root>'
exit(1)
t_file_out = File(sys.argv[2], 'recreate')
t_rate = []
for idx in xrange(25):
tree = Tree('t_rate_ct_%02d' % (idx + 1), 'rate of module CT_%02d' % (idx + 1))
tree.create_branches({'time_sec': 'D', 'cnts_ps': 'F[64]'})
t_rate.append(tree)
t_file_in = File(sys.argv[1], 'read')
t_modules = t_file_in.get('t_modules')
t_modules.activate(['is_bad', 'ct_num', 'trigger_bit', 'time_second'], True)
#t_modules.create_buffer()
#### read and fill data #####
counts = []
for idx in xrange(25):
counts.append([0] * 64)
first_flag = [True] * 25
pre_time = [0] * 25
for i,entry in enumerate(t_modules):
def add_qcd_yield_unc(outdir, var, channel, mva, mtmetcut):
from os import listdir
from os.path import isfile, join
onlyfiles = [ f for f in listdir(outdir) if isfile(join(outdir,f)) ]
hists = dict()
integrals = dict()
for fname in onlyfiles:
#print "filename", fname
f = File(outdir+"/"+fname)
for root, dirs, items in f.walk():
for name in items:
h = f.Get(join(root, name))
h.SetDirectory(0)
"""if fname.endswith("DATA.root"):
hists_data.append(h)
continue
elif fname.startswith("Single"):
hists_qcd.append(h)
continue"""
if not name in hists:
hists[name] = []
integrals[name] = 0.
hists[name].append((fname, h))
integrals[name] += h.Integral()
#hists[name].SetTitle(name)
f.Close()
variations = [("other", "QCD_fraction__down"), ("other", "QCD_fraction__up")
, ("other", "s_chan_fraction__down"), ("other", "s_chan_fraction__up")
, ("other", "tW_chan_fraction__up"), ("other", "tW_chan_fraction__down")
, ("wzjets", "Dibosons_fraction__up"), ("wzjets", "Dibosons_fraction__down")
, ("wzjets", "DYJets_fraction__up"), ("wzjets", "DYJets_fraction__down")
]
hists_var = {}
for (var_component, var) in variations:
#print var
for name in hists:
if name.endswith(var_component):
new_integral = 0
integral = integrals[name]
integral_orig = integral
#print "start", integral, integral_orig
if not name+"__"+var in hists_var:
hists_var[name+"__"+var] = []
for (fname,hist_orig) in hists[name]:
if check_starts(var, fname):
hist = hist_orig.Clone()
integral_orig = integral_orig - hist.Integral()
if var.endswith("down"):
scale = 0.5
if var.startswith("QCD"):
scale = 0.0
elif var.endswith("up"):
scale = 1.5
if var.startswith("QCD"):
scale = 2.0
hist.Scale(scale)
integral = integral - hist.Integral()
#print fname, hist.Integral()
#print "sub", integral, integral_orig
new_integral += hist.Integral()
hist.SetNameTitle(hist.GetName()+"__"+var, hist.GetTitle()+"__"+var)
hists_var[name+"__"+var].append(hist)
for (fname, hist_orig) in hists[name]:
if not check_starts(var, fname):
hist = hist_orig.Clone()
if integral_orig>0:
#print "SF", integral/integral_orig
hist.Scale(integral/integral_orig)
#print fname, hist.Integral()
new_integral += hist.Integral()
hist.SetNameTitle(hist.GetName()+"__"+var, hist.GetTitle()+"__"+var)
hists_var[name+"__"+var].append(hist)
#for (fname,hist) in hists[name]:
#print "new int", new_integral
#for (fname,hist) in hists[name]:
# print fname, hist.Integral()
elif not name.endswith("DATA") and len(name.split("__"))==2: #only add unc for nominal
if not name+"__"+var in hists_var:
hists_var[name+"__"+var] = []
for (fname, hist_orig) in hists[name]:
hist = hist_orig.Clone()
hist.SetNameTitle(hist.GetName()+"__"+var, hist.GetTitle()+"__"+var)
hists_var[name+"__"+var].append(hist)
return hists_var
import sys
from os.path import basename
from rootpy.io import File
from rootpy.tree import Tree
from rootpy.matrix import Matrix
if len(sys.argv) < 4:
print "USAGE: " + basename(sys.argv[0]) + " <time_win_mat.root> <decoded_file.root> <merged_file.root>"
exit(1)
time_win_filename = sys.argv[1]
decoded_data_filename = sys.argv[2]
merged_filename = sys.argv[3]
t_file_time_win = File(time_win_filename, "read")
begin_time_mat = t_file_time_win.get("begin_time_mat")
end_time_mat = t_file_time_win.get("end_time_mat")
max_count_mat = t_file_time_win.get("max_count_mat")
t_file_time_win.close()
t_file_merged_out = File(merged_filename, "recreate")
t_beam_event_tree = Tree("t_beam_event", "Beam Event Data")
t_beam_event_tree.create_branches({
'type': 'I',
'trig_accepted': 'B[25]',
'time_aligned': 'B[25]',
'pkt_count': 'I',
'lost_count': 'I',
'trigger_bit': 'B[1600]',
'trigger_n': 'I',
'''
Created on 11 Mar 2015
@author: kreczko
'''
from rootpy.io import File
from rootpy.plotting import Hist
from rootpy import asrootpy
rootpy_hist = Hist( 100, 0, 100, type = 'F' )
rootpy_hist.SetName('hist')
test_file = File('test.root', 'RECREATE')
test_file.mkdir("test")
test_file.cd('test')
rootpy_hist.Write()
test_file.Write()
test_file.Close()
read_file = File('test.root')
folder = read_file.Get('test')
hist = folder.hist
print hist.TYPE
read_file.Close()
hist = None
read_file = File('test.root')
hist = read_file.Get('test/hist')
histograms[1][idx].Draw('hist same')
ratio_histograms[idx].SetFillColor(kBlack)
ratio_histograms[idx].Scale(1000.0/ratio_histograms[idx].Integral())
for nbin in range(1, ratio_histograms[idx].GetNbinsX() + 1):
ratio_histograms[idx].SetBinContent(nbin, ratio_histograms[idx].GetBinContent(nbin) + 80.0)
ratio_histograms[idx].Draw('e3 same')
legend = TLegend(0.75, 0.3, 0.85, 0.5)
legend.SetLineColor(0)
legend.SetFillStyle(0)
legend.AddEntry(histograms[0][idx], 'DsK', 'l')
legend.AddEntry(histograms[1][idx], 'Ds#pi', 'l')
legend.AddEntry(ratio_histograms[idx], 'ratio', 'l')
legend.Draw()
gPad.Update()
else:
ratio_histograms[idx].Draw('e1')
if doPrint and (idx == 0 or idx > 6):
canvas0.Print('plots/DsK_ratio_'+str(idx)+'_'+str(zoom)+'.png')
canvas0.Print('plots/DsK_ratio_'+str(idx)+'_'+str(zoom)+'.pdf')
canvas.Update()
if not zoom:
dfile = File('data/raw_ratios.root', 'recreate')
dfile.cd()
for hist in ratio_histograms:
dfile.WriteTObject(hist)
dfile.Close()
if doPrint:
canvas.Print('plots/DsK_ratio_grid_'+str(zoom)+'.png')
canvas.Print('plots/DsK_ratio_grid_'+str(zoom)+'.pdf')
def test_file_open():
fname = 'delete_me.root'
with File.open(fname, 'recreate'):
pass
os.unlink(fname)
def save_to_root_file(histograms, file_name):
output = File(file_name, "recreate")
output.cd()
for histogram in histograms:
histogram.Write()
output.close()
def add_histos(outdir, var, channel, mva, mtmetcut):
from os import listdir
from os.path import isfile, join
onlyfiles = [ f for f in listdir(outdir) if isfile(join(outdir,f)) ]
hists = dict()
hists_data = []
hists_qcd = []
for fname in onlyfiles:
f = File(outdir+"/"+fname)
for root, dirs, items in f.walk():
for name in items:
h = f.Get(join(root, name))
if fname.endswith("DATA.root"):
hists_data.append(h)
continue
elif fname.startswith("Single"):
hists_qcd.append(h)
continue
if not name in hists:
hists[name] = []
if h.GetEntries()>0:
print fname, "factor", h.Integral()/math.sqrt(h.GetEntries())
for bin in range(1, h.GetNbinsX()+1):
print bin, h.GetBinContent(bin), h.GetBinError(bin)
hists[name].append(h)
#hists[name].SetTitle(name)
hist_data = hists_data[0]
for i in range(1, len(hists_data)):
hist_data.Add(hists_data[i])
hist_qcd = hists_qcd[0]
print "data", "factor", hist_data.Integral()/math.sqrt(hist_data.GetEntries())
for i in range(1, len(hists_qcd)):
print "add", i
hist_qcd.Add(hists_qcd[i])
print "QCD", "factor", hist_qcd.Integral()/math.sqrt(hist_qcd.GetEntries())
for bin in range(1, hist_data.GetNbinsX()+1):
hist_data.SetBinError(bin, math.sqrt(hist_data.GetBinContent(bin)))
hist_qcd.SetBinError(bin, math.sqrt(hist_qcd.GetBinContent(bin)))
print bin, hist_qcd.GetBinContent(bin), hist_qcd.GetBinError(bin)
hist_qcd.Scale(get_qcd_scale_factor(var, channel, mva, mtmetcut))
for bin in range(1, hist_data.GetNbinsX()+1):
print bin, hist_qcd.GetBinContent(bin), hist_qcd.GetBinError(bin)
hists[hist_data.GetName()] = [hist_data]
print hist_qcd.GetName()
hists[hist_qcd.GetName()].append(hist_qcd)
outfile = File(outdir+"/"+generate_file_name(False), "recreate")
for category in hists:
factor = 1.0
total_hist=hists[category][0].Clone()
total_hist.Reset("ICE")
#total_hist.Sumw2()
total_hist.SetNameTitle(category,category)
outfile.cd()
print category
for bin in range(1, total_hist.GetNbinsX()+1):
zero_error = 0.
zero_integral = 0.
nonzero_error = 0.
bin_sum = 0.
#max_zero_error = 0.
for hist in hists[category]:
print "hist", hist.GetBinContent(bin), hist.GetBinError(bin)**2
if hist.GetEntries()>0:
#factor = hist.Integral()/math.sqrt(hist.GetEntries())
#print "fact", factor,hist.Integral(), hist.GetEntries()
min_nonzero_error = sys.float_info.max
for bin1 in range(1, hist.GetNbinsX()+1):
if hist.GetBinContent(bin1) > 0 and hist.GetBinError(bin1) < min_nonzero_error:
min_nonzero_error = hist.GetBinError(bin1)
print "error", min_nonzero_error
else:
min_nonzero_error = 0.
if hist.GetBinContent(bin) < 0.00001:
#zero_error += factor**2 * hist.Integral()
zero_error += min_nonzero_error**2
zero_integral += hist.Integral()
else:
bin_sum += hist.GetBinContent(bin)
nonzero_error += hist.GetBinError(bin)**2
#print "...hist", hist.GetBinContent(bin), hist.GetBinError(bin)**2
total_hist.SetBinContent(bin, bin_sum)
if zero_integral > 0:
#total_error = math.sqrt(nonzero_error + zero_error / zero_integral)
total_error = math.sqrt(nonzero_error + zero_error)
else:
total_error = math.sqrt(nonzero_error)
total_hist.SetBinError(bin, total_error)
print category, "bin", bin, "content", bin_sum, "error", total_error
print category, "bin", bin, "weight", total_error**2/bin_sum
total_hist.Write()
"""for category in hists: #Imitate hadd
#factor = 1.0
total_hist=hists[category][0].Clone()
total_hist.Reset("ICE")
print "cat", category
total_hist.SetNameTitle(category,category)
total_hist.Sumw2()
outfile.cd()
for hist in hists[category]:
print "hist", hist, hist.GetName()
factor = 1.0
if hist.GetEntries()>0:
factor = hist.Integral()/hist.GetEntries()
for bin in range(1, total_hist.GetNbinsX()+1):
if hist.GetBinError(bin) < factor:
hist.SetBinError(bin, factor)
#total_hist.SetBinContent(bin, total_hist.GetBinContent(bin) + hist.GetBinContent(bin))
#total_hist.SetBinError(bin, total_hist.GetBinError(bin) + hist.GetBinError(bin))
total_hist.Add(hist)
total_hist.Write()"""
outfile.Write()
outfile.Close()
#!/usr/bin/env python
import argparse
import csv
from rootpy import ROOT
from rootpy.io import File
from rootpy.tree import Tree
from conv_fun import *
parser = argparse.ArgumentParser(description='Convert platform parameters data from CSV file to ROOT file')
parser.add_argument("filename", help = "CSV file of platform parameters data")
parser.add_argument("-o", dest = "outfile", help = "ROOT file to store platform parameters data", default = "TG2_PPD_file.root")
args = parser.parse_args()
t_file_out = File(args.outfile, "recreate")
t_tree_ppd = Tree("t_ppd", "platform parameters data")
t_tree_ppd.create_branches({
"pitch_angle" : "D" ,
"yaw_angle" : "D" ,
"roll_angle" : "D" ,
"pitch_angle_v" : "D" ,
"yaw_angle_v" : "D" ,
"roll_angle_v" : "D" ,
"orbit_agl_v" : "D" ,
"longitude" : "D" ,
"latitude" : "D" ,
"geocentric_d" : "D" ,
"ship_time_sec" : "D" ,
"utc_time_sec" : "D" ,
"utc_time_str" : "C[32]" ,
"flag_of_pos" : "I" ,
from rootpy.io import File
if __name__=="__main__":
fi1 = "efficiency.root"
fi2 = "efficiency_working.root"
fi1 = File(fi1)
fi2 = File(fi2)
for root, dirs, items in fi1.walk():
for it in items:
it1 = fi1.Get("/".join([root,it]))
it2 = fi2.Get("/".join([root,it]))
print it1, it2
i = 0
ov1 = it1.overflow()
ov2 = it2.overflow()
un1 = it1.underflow()
un2 = it2.underflow()
if ov1+ov2>0 and abs(ov1-ov2)/(ov1+ov2):
s += "ooo"
if un1+un2>0 and abs(un1-un2)/(un1+un2):
s += "uuu"
print ov1, ov2, un1, un2
for a, b, c, d in zip(list(it1.y()), list(it2.y()), list(it1.yerravg()), list(it2.yerravg()), ):
s = ""
x1=0
x2=0
def main():
set_root_defaults()
# prevent directory ownership of ROOT histograms (python does the garbage collection)
TH1F.AddDirectory( False )
parser = OptionParser()
parser.add_option( "-n", "--n_toy_mc",
dest = "n_toy_mc", default = 300,
help = "number of toy MC to create", type = int )
parser.add_option( "-o", "--output",
dest = "output_folder", default = 'data/toy_mc/',
help = "output folder for toy MC" )
parser.add_option( "-v", "--variable", dest = "variable", default = 'MET',
help = "set the variable to analyse (MET, HT, ST, MT, WPT)" )
parser.add_option( "-m", "--metType", dest = "metType", default = 'type1',
help = "set MET type for analysis of MET, ST or MT" )
parser.add_option( "-c", "--centre-of-mass-energy", dest = "CoM", default = 8,
help = "set the centre of mass energy for analysis. Default = 8 [TeV]", type = int )
parser.add_option( '-V', '--verbose', dest = "verbose", action = "store_true",
help = "Print the event number, reco and gen variable value" )
( options, _ ) = parser.parse_args()
measurement_config = XSectionConfig( options.CoM )
centre_of_mass = options.CoM
ttbar_xsection = measurement_config.ttbar_xsection
variable = options.variable
met_type = measurement_config.translate_options[options.metType]
n_toy_mc = options.n_toy_mc
make_folder_if_not_exists( options.output_folder )
# get histograms
input_file_hists = File( measurement_config.unfolding_madgraph )
# define output file
out_file_template = '%s/toy_mc_%s_N_%d_%dTeV.root'
out_file_name = out_file_template % (options.output_folder, variable, n_toy_mc, centre_of_mass)
output = File( out_file_name, 'recreate' )
for channel in ['electron', 'muon']:
# first get the weights
h_truth, h_measured, h_response, _ = get_unfold_histogram_tuple( input_file_hists,
variable,
channel,
met_type,
centre_of_mass,
ttbar_xsection,
load_fakes = False )
# create directories
directory = output.mkdir( channel )
mkdir = directory.mkdir
cd = directory.cd
cd()
# generate toy MC
for i in range( 1, n_toy_mc + 1 ):
mkdir( 'toy_%d' % i )
cd( 'toy_%d' % i )
# create histograms
# add tuples (truth, measured, response) of histograms
truth = generate_toy_MC_from_distribution(h_truth)
measured = generate_toy_MC_from_distribution(h_measured)
response = generate_toy_MC_from_2Ddistribution(h_response)
truth.SetName('truth')
measured.SetName('measured')
response.SetName('response')
truth.Write()
measured.Write()
response.Write()
output.Write()
output.Close()
