def __init__(self, POIs=None, file_names=None, combine_method=None):
self.log = Logger().getLogger(self.__class__.__name__, 10)
self.combine_method = combine_method
self.combine_result = None
self.global_best_fit = {} #best fit from all the files.
self.global_best_fit_dict = {} #contains one best fit per input file
self.ll_values_dict = {}
self.ul_values_dict = {}
self.contours = {}
self.contour_graph = {}
self._has_parsed_combine_result_already = False
self.set_POI(POIs)
self.set_files(file_names)
def __init__(self):
"""Initialize whatever is needed"""
self.my_logger = Logger()
self.log = self.my_logger.getLogger(self.__class__.__name__, 10)
# }
self.DEBUG = self.my_logger.is_debug()
self.pp = pprint.PrettyPrinter(indent=4)
# initialize RooFit
gSystem.Load("libHiggsAnalysisCombinedLimit.so")
self.output_filename = "worskapce_with_embedded_toys.root"
def __init__(self,name = "limitVsLumi_plot" ):
self.log = Logger().getLogger(self.__class__.__name__, 10)
self.name = name
#ROOT.gSystem.AddIncludePath("-I$ROOFITSYS/include/");
ROOT.gROOT.ProcessLine(".L tdrstyle.cc")
from ROOT import setTDRStyle
ROOT.setTDRStyle(True)
ROOT.gStyle.SetPalette(1)
ROOT.gStyle.SetOptStat(0)
#self.copy_to_web_dir = False
#self.webdir = ""
self.pp = pprint.PrettyPrinter(indent=4)
class ToyDataSetManager(RootHelperBase):
def __init__(self):
"""Initialize whatever is needed"""
self.my_logger = Logger()
self.log = self.my_logger.getLogger(self.__class__.__name__, 10)
# }
self.DEBUG = self.my_logger.is_debug()
self.pp = pprint.PrettyPrinter(indent=4)
# initialize RooFit
gSystem.Load("libHiggsAnalysisCombinedLimit.so")
self.output_filename = "worskapce_with_embedded_toys.root"
def set_toys_path(self, toys_path):
"""
Set the path for the toy dataset.There is aleays one active toy held in self.toys.
"""
self.toys_path = toys_path
self.toys = self.get_object(path=toys_path, object_type=RooAbsData, clone=False)
def set_workspace_path(self, ws_path):
"""
Set the path for the workspace where toys will be included.
There is only one workspace that can be active in the class.
"""
self.ws = self.get_object(path=ws_path, object_type=RooWorkspace, clone=False)
def set_output_file_name(self, output_filename):
"""
Set the name of the output root file.
"""
self.output_filename = output_filename
def set_new_toys_name(self, new_toys_name):
"""
Set name for toys in the workspace
"""
self.new_toys_name = new_toys_name
def import_toys_to_ws(self, ws_path=None, toys_path=None, output_filename=None, new_toys_name=None):
"""
Imports a given toys dataset (or multiple toys) into the workspace and dumps to new root file.
Parameters:
-----------
ws_path : path to exisitng workspace (string)
toys_path : path or list of paths to toys.TODO add regexp parsing to import matching toys.
output_filename : file name of the output workspace
new_toys_name : in case of one toy import, a new name can be set. In case of list, the name is set
to be the same as in the source file.
Returns:
--------
Returns 0 in case it goes trough without erorrs(?).
"""
# TODO set checks for the input provided
if ws_path:
self.set_workspace_path(ws_path)
if output_filename:
self.set_output_file_name(output_filename)
if new_toys_name:
self.set_new_toys_name(new_toys_name)
try:
self.ws
except AttributeError:
raise AttributeError, "You need to provide workspace path."
if toys_path:
toys_path_list = []
if isinstance(toys_path, list):
toys_path_list = toys_path
elif isinstance(toys_path, str):
toys_path_list = [toys_path]
for the_toy in toys_path_list:
self.set_toys_path(the_toy)
toys_name = self.get_paths(the_toy)[-1] # just getthe name of toys object in the root file.
self.log.info("Setting toys name in workspace to: {0}".format(toys_name))
self.set_new_toys_name(toys_name)
self.toys.SetName(self.new_toys_name)
getattr(self.ws, "import")(self.toys)
self.log.info(
"Imported DataSet '{0}' into workspace '{1}'.".format(self.toys.GetName(), self.ws.GetName())
)
else:
try:
self.toys
except AttributeError:
raise AttributeError, "You need to provide toys path."
try:
self.new_toys_name
except AttributeError:
toys_name = self.get_paths(self.toys_path)[-1] # just getthe name of toys object in the root file.
self.log.info("Setting toys name in workspace to: {0}".format(toys_name))
self.set_new_toys_name(toys_name)
self.toys.SetName(self.new_toys_name)
getattr(self.ws, "import")(self.toys)
self.log.info("Imported DataSet '{0}' into workspace '{1}'.".format(self.toys.GetName(), self.ws.GetName()))
self.ws.data(self.toys.GetName()).Print()
self.ws.data(self.toys.GetName()).Print("v")
# write workspace
self.ws.writeToFile(self.output_filename)
self.log.info("Writing workspace '{0}' to file {1}".format(self.ws.GetName(), self.output_filename))
return 0
def set_dataset_name(self, dataset_name):
"""
Set name of the dataset in workspace.
"""
self.dataset_name = dataset_name
def import_dataset_to_ws(self, dataset, workspace, output_filename=None, new_name=None):
"""
Import dataset to worspace workspace.
"""
if new_name:
dataset.SetName(new_name)
if output_filename:
self.set_output_file_name(output_filename)
self.log.info(
"Imported DataSet '{0}' into workspace '{1}' and written to file {2}.".format(
dataset.GetName(), workspace.GetName(), self.output_filename
)
)
pass
def set_workspace(self, workspace):
"""
Provide workspace from path naload it to self.ws or
provide directly workspace and load it to self.ws
"""
if isinstance(workspace, RooWorkspace):
self.ws = workspace
self.log.debug("Loaded in workspace {0}.".format(self.ws.GetName()))
elif isinstance(workspace, str):
self.set_workspace_path(self, workspace)
self.log.debug("Loaded in workspace {0} from path: ".format(workspace))
def dump_datasets_to_file(self, output_filename=None, access="RECREATE"):
"""
Write all datasets collected in the basket(RootHelperBase) to a file.
"""
if output_filename:
self.set_output_file_name(output_filename)
self.dump_basket_to_file(self.output_filename, access)
self.log.info("All items from the basket have been written to file: {0}".format(self.output_filename))
return 0
def get_dataset_from_tree(
self,
path_to_tree,
tree_variables,
weight="1==1",
weight_var_name=0,
dataset_name="my_dataset",
basket=True,
category=None,
):
"""
Creates RooDataSet from a plain root tree given:
- variables name list
- weight expression. It works in the same way as TTree cut.
Returns:
--------
- RooDataSet
- also fills the basket with datasets (basket inhereted from RootHelperBase class)
TODO
----
- add implementation for category setting(check in prepare_toy_datasets_for_sync)
- check if adding toy dataset to each channel workspace individually behaves well
after combineCards.py.
"""
# make RooRealVars from tree_variables
my_arg_set = RooArgSet()
my_rrv = dict()
for var_name in tree_variables:
# TODO implement check that branch exist
my_rrv[var_name] = RooRealVar(var_name, var_name, -999999999, 999999999)
my_arg_set.add(my_rrv[var_name])
if self.DEBUG:
self.log.debug("RooArgSet is now:")
my_arg_set.Print()
# get the tree from path_to_tree
my_tree = self.get_TTree(path_to_tree, cut=weight)
self.log.debug("Selected tree contains {0} events".format(my_tree.GetEntries()))
# create RooDataSet and reduce tree if needed
# self.dataset_from_tree = RooDataSet(dataset_name, dataset_name, my_tree, my_arg_set, weight).reduce(my_arg_set)
self.dataset_from_tree = RooDataSet(dataset_name, dataset_name, my_tree, my_arg_set)
# self.dataset_from_tree = RooDataSet(dataset_name, dataset_name, my_tree, my_arg_set, "", weight_var_name)
# data[j]=new RooDataSet(Form("data%d",j),Form("data%d",j),outTree,RooArgSet(rCMS_zz4l_widthKD,rCMS_zz4l_widthMass,rweightFit),"","_weight_");
self.log.debug("RooDataSet contains {0} events".format(self.dataset_from_tree.sumEntries()))
# .reduce(ROOT.RooArgSet(self.D0))
self.current_arg_set = my_arg_set
# add dataset to basket
if basket:
self.add_to_basket(self.dataset_from_tree, new_name=dataset_name, new_title=dataset_name)
return self.dataset_from_tree
def get_current_arg_set(self):
"""
Return last dataset setup used by get_dataset_from_tree().
"""
return self.current_arg_set
class FitResultReader(object):
"""Reads the tree with fit information and gives back the
information relevant for plotng the limits or measurements.
"""
def __init__(self, POIs=None, file_names=None, combine_method=None):
self.log = Logger().getLogger(self.__class__.__name__, 10)
self.combine_method = combine_method
self.combine_result = None
self.global_best_fit = {} #best fit from all the files.
self.global_best_fit_dict = {} #contains one best fit per input file
self.ll_values_dict = {}
self.ul_values_dict = {}
self.contours = {}
self.contour_graph = {}
self._has_parsed_combine_result_already = False
self.set_POI(POIs)
self.set_files(file_names)
def set_POI(self, POIs):
"""Provide a list of POIs in terms of python list
of strings or string with POIs separated by ";:*, "
"""
self.POI = []
assert isinstance(POIs, list) or isinstance(
POIs, str
), "POIs should be provided either as list of strings or as string with \";: \" as delimiters. "
if isinstance(POIs, list):
self.POI = POIs
elif isinstance(POIs, str):
import re
POIs = re.sub('[;:, ]+', ':',
POIs) #pois can be split by ";:*, " - we don't care
self.POI = POIs.split(":")
for poi in self.POI:
self.global_best_fit.update({poi: None})
self.log.debug('Initializing the best fit dictionary {0}'.format(
self.global_best_fit))
self.log.debug('Setting POI list to {0}'.format(self.POI))
def set_files(self, file_names, start_dir="."):
"""Set the list of output files from combine that will be used.
One can use the start dir and filename pattern to run on all
files that are found recursively on the path.
"""
self.file_list = []
assert isinstance(file_names, list) or isinstance(
file_names, str
), "File names should be provided either as list of strings or as string with \";,: \" as delimiters. "
assert isinstance(
start_dir,
str), "The start directory should be provided as string."
if isinstance(file_names, list):
self.file_list = file_names
elif isinstance(file_names, str):
raise ValueError, 'Please provide a list of strings for the file names. The option with strings only doesn\'t work for the moment. :('
self.file_list = return_filenames(start_dir, self.file_list)
print 'File list = ', self.file_list
self.log.debug('Loaded {0} files.'.format(len(self.file_list)))
self._has_parsed_combine_result_already = False #has to be set to False so that the limits and best fits are recalculated when new file is set.
def _get_crossings_for_limits(self, list_of_segments, cl=0.68):
"""Internal function for getting the Y values from given
TGraph objets. It is used to get POI limits for particular
confidence level.
"""
assert belongsTo(
float(cl), 0,
1), "Confidence level has to be given in interval [0,1]"
quantileExpected = 1 - cl
values = []
for seg in list_of_segments:
#ll_seg is a TGraph
xmin = TMath.MinElement(seg.GetN(), seg.GetX())
xmax = TMath.MaxElement(seg.GetN(), seg.GetX())
if belongsTo(quantileExpected, xmin, xmax):
values.append(seg.Eval(quantileExpected))
return values
def get_graph(self,
contour_axis="x:y:z",
dims=1,
y_offset=0.0,
z_offset=0.0):
"""Returns the full likelihood scan graph.
Specify contour_axis= "2*deltaNLL" or "1-quantileExpected"
The last axis provided should be
"""
import re
contour_axis = re.sub(
'[;:]+', ':', contour_axis) #can be split by ";: " - we don't care
try:
self.contour_graph[contour_axis]
except KeyError:
contour_axis_list = contour_axis.split(":")
dims = len(contour_axis_list)
assert 1 < dims <= 3, "We can accept 2 to 3 axis for the graph. You provided {0}.Please behave :)".format(
dims)
assert (
'deltaNLL' in contour_axis_list[-1]
or 'quantileExpected' in contour_axis_list[-1]
), 'Your last axis has to contain either deltaNLL or quantileExpected.'
import copy
#required_branches = copy.deepcopy(contour_axis_list)
required_branches = []
#Solve to accept even a formula as the axis.
if 'deltaNLL' in contour_axis_list[-1]:
#self.log.debug('deltaNLL in contour_axis_list: {0}'.format(contour_axis_list[-1]) )
contour_axis_list[-1] = str(contour_axis_list[-1]).replace(
'deltaNLL', 't.deltaNLL')
#required_branches[-1] = 'deltaNLL'
required_branches.append('deltaNLL')
#self.log.debug('deltaNLL is an estimator: {0}'.format(contour_axis_list[-1]) )
elif 'quantileExpected' in contour_axis_list[-1]:
contour_axis_list[-1] = contour_axis_list[-1].replace(
'quantileExpected', 't.quantileExpected')
#required_branches[-1] = 'quantileExpected'
required_branches.append('quantileExpected')
#self.log.debug('quantileExpected is an estimator: {0}'.format(contour_axis_list[-1]) )
self.log.debug(
'Changing names of pois for evaluation of formula later: N_poi = {0} N_axis= {1}'
.format(len(self.POI), len(contour_axis_list[:-1])))
for poi_id in range(len(self.POI)):
for axis_id in range(len(contour_axis_list[:-1])):
self.log.debug(
'Changing names of pois for evaluation of formula later: poi_id = {0} axis_id = {1}'
.format(poi_id, axis_id))
if self.POI[poi_id] == 'r':
contour_axis_list[axis_id] = 't.r'
else:
contour_axis_list[axis_id] = contour_axis_list[
axis_id].replace(self.POI[poi_id],
't.{0}'.format(self.POI[poi_id]))
#required_branches[axis_id] = self.POI[poi_id]
required_branches.append(self.POI[poi_id])
self.log.debug(
'Contour axis list changed for evaluation of formula to {0}'.
format(contour_axis_list))
if dims == 2:
self.contour_graph[contour_axis] = TGraph()
elif dims == 3:
self.contour_graph[contour_axis] = TGraph2D()
self.contour_graph[contour_axis].SetNameTitle(
contour_axis, contour_axis.replace(':', ';'))
self.log.debug(
'Graph {0} is being created from the tree in {1}'.format(
contour_axis, self.file_list[0]))
rootfile = TFile.Open(self.file_list[0], 'READ')
if not rootfile:
raise IOError, 'The file {0} either doesn\'t exist or cannot be open'.format(
self.file_list[0])
t = rootfile.Get('limit')
required_branches = list(set(required_branches))
self.log.debug(
'Required branches are : {0}'.format(required_branches))
for axis in range(dims):
assert t.GetListOfBranches().FindObject(
required_branches[axis]
), "The branch \"{0}\" doesn't exist.".format(
required_branches[axis])
t.SetBranchStatus("*", False)
#for axis in range(dims):
#t.SetBranchStatus(required_branches[axis], True)
for branch in required_branches:
t.SetBranchStatus(branch, True)
#x_y_z_list = []
x_y_z_set = set()
x_y_set = set()
#x_y_dict= dict()
for en in range(1, t.GetEntriesFast()):
t.GetEntry(en)
if AreSame(t.quantileExpected, 1):
#skip all the global fit entries (case when hadding scan outputs)
self.log.debug(
"This entry ({0}) is coming from global fit. Skipping."
.format(en))
continue
if dims == 2:
X = eval(contour_axis_list[0])
Y = eval(contour_axis_list[1])
x_y_set.add((X, Y))
if en % 100 == 0:
#self.log.debug('Inputs X={0} Y={1}'.format(eval('t.{0}'.format(required_branches[0])),eval('t.{0}'.format(required_branches[1]))))
self.log.debug('Entry={2} X={0} Y={1}'.format(
X, Y, en))
#self.contour_graph[contour_axis].SetPoint(en-1,X,Y)
elif dims == 3:
X = eval(contour_axis_list[0])
Y = eval(contour_axis_list[1])
Z = eval(contour_axis_list[2])
#x_y_z_list.append((X,Y,Z))
if en % 100 == 0: #FIXME WARNING This is just to read less points
x_y_z_set.add((X, Y, Z))
if en % 1000 == 0:
#t.Show()
self.log.debug('Entry={3} X={0} Y={1} Z={2}'.format(
X, Y, Z, en))
#self.contour_graph[contour_axis].SetPoint(en-1,X,Y,Z)
self.log.debug('Setting points of graph from sorted lists.')
if dims == 2:
i_elem = 0
for point in sorted(list(x_y_set)):
self.contour_graph[contour_axis].SetPoint(
i_elem, point[0], point[1] + y_offset)
i_elem += 1
elif dims == 3:
i_elem = 0
for point in sorted(list(x_y_z_set)):
self.contour_graph[contour_axis].SetPoint(
i_elem, point[0], point[1] + y_offset,
point[2] + z_offset)
i_elem += 1
del x_y_set
del x_y_z_set
#if dims==3:
##TGraph2D is too slow (Delunay triangulation) - we want to give back a TH2D
#th2d = self.contour_graph[contour_axis].GetHistogram("empty")
#self.log.debug("Copyng TGraph2D to TH2D EMPTY histo with nx = {0}, ny = {1}".format(th2d.GetXaxis().GetNbins(),th2d.GetYaxis().GetNbins() ))
#for point in x_y_z_list:
#th2d.SetBinContent(th2d.GetXaxis().FindBin(point[0]),th2d.GetYaxis().FindBin(point[1]),point[2])
##self.log.debug('TH2D filled with value={0}. Current entries = {1}'.format(point, th2d.GetEntries()))
#import copy
#self.contour_graph[contour_axis] = copy.deepcopy(th2d)
#self.log.debug('TH2D given to contour {0} of type {1}'.format(self.contour_graph[contour_axis], type(self.contour_graph[contour_axis])))
self.log.debug('Returning filled graph.')
return self.contour_graph[contour_axis]
def ll_values(self, POI, cl=0.68):
"""returns a list of lower limits for a given level for a given POI
"""
if not self._has_parsed_combine_result_already:
self._parse_combine_result()
assert belongsTo(
float(cl), 0,
1), "Confidence level has to be given in interval [0,1]"
cl_name = "{1}_CL@{0:.2f}".format(cl, POI)
try:
self.ll_values_dict[cl_name]
except KeyError:
self.ll_values_dict[cl_name] = self._get_crossings_for_limits(
self.raising_segments[POI], float(cl))
self.log.debug('Creating limit for C.L.@{0}'.format(cl))
else:
self.log.debug('Returning existing limit for C.L.@{0}'.format(cl))
return self.ll_values_dict[cl_name]
def ul_values(self, POI, cl=0.68):
"""returns a list of lower limits for a given level for a given POI
"""
if not self._has_parsed_combine_result_already:
self._parse_combine_result()
assert belongsTo(
float(cl), 0,
1), "Confidence level has to be given in interval [0,1]"
cl_name = "{1}_CL@{0:.2f}".format(cl, POI)
try:
self.ul_values_dict[cl_name]
except KeyError:
self.ul_values_dict[cl_name] = self._get_crossings_for_limits(
self.falling_segments[POI], float(cl))
self.log.debug('Creating limit for C.L.@{0}'.format(cl))
else:
self.log.debug('Returning existing limit for C.L.@{0}'.format(cl))
return self.ul_values_dict[cl_name]
def best_fit(self, POI):
"""Get the best fit value fora particular POI
"""
if not self._has_parsed_combine_result_already:
self._parse_combine_result()
try:
self.global_best_fit[POI]
except KeyError:
raise KeyError, 'The POI name \"{0}\" is invalid.'.format(POI)
else:
return float(self.global_best_fit[POI])
def is_set_ll(self, POI, cl=0.68):
assert belongsTo(
float(cl), 0,
1), "Confidence level has to be given in interval [0,1]"
cl_name = "{1}_CL@{0:.2f}".format(cl, POI)
try:
self.ll_values_dict[cl_name]
except KeyError:
raise KeyError, 'The POI name \"{0}\" is invalid.'.format(POI)
else:
return (len(self.ll_values_dict[cl_name]) > 0)
def is_set_ul(self, POI, cl=0.68):
assert belongsTo(
float(cl), 0,
1), "Confidence level has to be given in interval [0,1]"
cl_name = "{1}_CL@{0:.2f}".format(cl, POI)
try:
self.ul_values_dict[cl_name]
except KeyError:
raise KeyError, 'The POI name \"{0}\" is invalid.'.format(POI)
else:
return (len(self.ul_values_dict[cl_name]) > 0)
def is_set_best_fit(self, POI):
try:
self.global_best_fit[POI]
except KeyError:
raise KeyError, 'The POI name \"{0}\" is invalid.'.format(POI)
else:
return (self.global_best_fit[POI] != None)
def get_results_dict(self,
POI,
option='standard',
rescale_expression='',
invert_LL_UL=False):
"""Returns a dict with best fit values and limits at 68(95)%
"""
self.log.info('Compiling the fit results dictionary...')
if option.lower() == 'standard':
#import collections
#self.limits_dict = collections.OrderedDict()
self.limits_dict = {}
self.limits_dict['BF'] = self.best_fit(POI)
self.limits_dict['LL68'] = self.ll_values(POI, 0.68)
self.limits_dict['LL95'] = self.ll_values(POI, 0.95)
self.limits_dict['UL68'] = self.ul_values(POI, 0.68)
self.limits_dict['UL95'] = self.ul_values(POI, 0.95)
self.log.debug('Limits are: {0}'.format(self.limits_dict))
import copy
return_dict = copy.deepcopy(
self.limits_dict
) #because dict is mutable... we don't want the initial dict to be changed
if POI in rescale_expression: #the rescale must contain the formula with the POI string inside
for key in return_dict.keys():
if isinstance(return_dict[key], float):
the_value = return_dict[key]
return_dict[key] = eval(
rescale_expression.replace(POI,
str(return_dict[key])))
self.log.debug(
'Rescaling {3} value with {0}: {1} ---> {2}'.
format(rescale_expression, the_value,
return_dict[key], key))
elif isinstance(return_dict[key], list):
for val_i in range(len(return_dict[key])):
the_value = return_dict[key][val_i]
return_dict[key][val_i] = eval(
rescale_expression.replace(
POI, str(return_dict[key][val_i])))
self.log.debug(
'Rescaling {3} value with {0}: {1} ---> {2}'.
format(rescale_expression, the_value,
return_dict[key][val_i], key))
if invert_LL_UL:
return_dict['UL68'], return_dict['LL68'] = return_dict[
'LL68'], return_dict['UL68']
return_dict['UL95'], return_dict['LL95'] = return_dict[
'LL95'], return_dict['UL95']
return return_dict
else:
raise RuntimeError, 'The option {0} is still not implemented. Do you want to volonteer? :)'.format(
option)
def get_contours(self, contour_axis, limits=None):
"""Return dict of lists of TGraph contours with a given confidence level.
The keys are levels...
The code taken from http://root.cern.ch/root/html534/tutorials/hist/ContourList.C.html
"""
self.log.debug('Extracting contours for {0} at levels {1}'.format(
contour_axis, limits))
if limits == None:
limits = ['0.68', '0.95'] #default limit values
import re
import copy
contour_axis = re.sub(
'[;:]+', ':', contour_axis) #can be split by ";: " - we don't care
n_missing = 0
for limit in limits:
try:
#if the contours exist, we will return them imediatelly
self.contours[contour_axis][str(limit)]
#except KeyError:
except:
n_missing += 1
self.get_graph(contour_axis)
if n_missing == 0:
self.log.debug('Contour exist. Returning.')
return self.contours[contour_axis]
else:
#initialize contours
self.contours[contour_axis] = {}
#for level in limits:
#self.contours[str(level)]=[]
self.log.debug('Contours before extracting {0}'.format(self.contours))
graph_contours = self.contour_graph[contour_axis].Clone(
"graph_contours")
self.log.debug('Contour is of type {0}'.format(type(graph_contours)))
#import array
contours = array('d', [float(lim) for lim in limits])
#if isinstance(graph_contours,plotLimit.TH2D):
if 'TH2D' in str(type(graph_contours)):
graph_contours.SetContour(len(contours), contours)
c = TCanvas("c", "Contour List", 0, 0, 600, 600)
c.cd()
graph_contours.Draw("CONT Z LIST")
c.Update(
) #// Needed to force the plotting and retrieve the contours in TGraphs
conts = gROOT.GetListOfSpecials().FindObject("contours")
#// Get Contours
#conts = gROOT.GetListOfSpecials().FindObject("contours")
contLevel = None
curv = None
TotalConts = 0
if (conts == None):
print "*** No Contours Were Extracted!\n"
TotalConts = 0
return
else:
TotalConts = conts.GetSize()
print "TotalConts = %d\n" % (TotalConts)
#tgraph_list = {}
#self.contours[contour_axis]
for i in reversed(range(0, TotalConts)):
#last contour is the first in the contour array
contLevel = conts.At(i)
self.contours[contour_axis][str(limits[i])] = []
print "Contour %d has %d Graphs\n" % (i, contLevel.GetSize())
for j in range(0, contLevel.GetSize()):
#tgraph_list.append(copy.deepcopy(contLevel.At(j)))
self.contours[contour_axis][str(limits[i])].append(
copy.deepcopy(contLevel.At(j)))
#elif isinstance(graph_contours,plotLimit.TGraph2D):
#elif 'TGraph2D' in str(type(graph_contours)):
## Create a struct
##import string
##limits_string = string.join(limits,',')
##gROOT.ProcessLine("Float_t MyContourLevels[] = {{{0}}}".format(string.join(limits,',')))
##from ROOT import MyContourLevels
##Create branches in the
#c = TCanvas("c_tgraph2D","Contour List",0,0,600,600)
#graph_contours.Draw("COLZ")
###c.Update()
#for limit in limits:
#self.log.debug('Doing the contours for {0}'.format(limit))
#conts = graph_contours.GetContourList(float(limit))
#for j in range(0,conts.GetSize()):
#self.log.debug('Adding contour: level={0} i={1} '.format(limit,j))
#self.contours[contour_axis][str(limit)].append(copy.deepcopy(conts.At(j)))
c = TCanvas("c_tgraph2D", "Contour List", 0, 0, 600, 600)
graph_contours.Draw("COLZ")
c.Update()
for limit in limits:
conts = graph_contours.GetContourList(float(limit))
self.log.debug(
'Doing the contours for {0}: #contours = {1}'.format(
limit, conts.GetSize()))
self.contours[contour_axis][str(limit)] = []
for j in range(0, conts.GetSize()):
self.log.debug('Adding contour: level={0} i={1} '.format(
limit, j))
self.contours[contour_axis][str(limit)].append(
copy.deepcopy(conts.At(j)))
self.log.debug('Contour for {0} is of type={1}.'.format(
contour_axis, type(self.contours[contour_axis])))
print self.contours[contour_axis]
#we return dict with keys=limits and values=lists of TGraph objects
return self.contours[contour_axis]
def set_combine_method(self, combine_method):
"""Set method in order to know how the limit trees look like.
"""
self.combine_method = combine_method
def _get_TGraph_from_segment(self, segment):
"""Create TGraph from list of tuples(qe, poi, dNLL)
"""
qe_vals = array('d', [t[0] for t in segment])
poi_vals = array('d', [t[1] for t in segment])
return TGraph(len(segment), qe_vals, poi_vals)
def _parse_combine_result(self, combine_method="MultiDimFit"):
"""Parsing the combine result and filling the contour information.
Should be run on first demand of any information.
"""
self._has_parsed_combine_result_already = True
self.log.debug(
"Parsing the combine output files for method = {0}".format(
combine_method))
assert len(self.file_list) > 0, "There is no files to read."
#assert len(self.file_list)==1, "Currently, we allow only one file from combine output. The implementation is still lacking this feature..."
#containers for TGraph raising and falling segments
self.falling_segments = {poi: [] for poi in self.POI}
self.raising_segments = {poi: [] for poi in self.POI}
for poi in self.POI:
for root_file_name in self.file_list:
root_file_name = self.file_list[0]
#get the TTree and enable relevant branches (POI, quantileExpected, deltaNLL)
self.log.debug("Parsing the combine output file = {0}".format(
root_file_name))
rootfile = ROOT.TFile.Open(root_file_name, 'READ')
if not rootfile:
raise IOError, 'The file {0} either doesn\'t exist or cannot be open'.format(
root_file_name)
t = rootfile.Get('limit')
assert t.GetListOfBranches().FindObject(
poi), "The branch \"{0}\" doesn't exist.".format(poi)
#don't read uninteresting branches
t.SetBranchStatus("*", False)
t.SetBranchStatus("quantileExpected", True)
t.SetBranchStatus(poi, True)
t.SetBranchStatus("deltaNLL", True)
#get the best fit
if self.global_best_fit[poi] == None:
t.GetEntry(0)
if AreSame(t.quantileExpected, 1) and AreSame(
t.deltaNLL, 0
): #This is true if combine has found a good minimum.
self.global_best_fit[poi] = eval('t.{0}'.format(poi))
self.global_best_fit_dict.update({
poi: {
'best_fit': eval('t.{0}'.format(poi)),
'quantileExpected': t.quantileExpected,
'2*deltaNLL': 2 * t.deltaNLL
}
})
self.log.debug(
"Global best fit in file {0} = {1}".format(
root_file_name, self.global_best_fit_dict))
is_raising = False
#we want to find the first trend of the function
ien = 1
tmp_list_qe_poi_dNLL = []
while True:
t.GetEntry(ien)
if ien == 1:
qe_prev = t.quantileExpected
if t.quantileExpected > qe_prev:
is_raising = True
break
elif t.quantileExpected < qe_prev:
is_raising = False
break
qe_prev = t.quantileExpected
ien += 1
self.log.debug(
'Detected trend of first interval: Raising = {0}, Falling = {1}'
.format(is_raising, (not is_raising)))
self.log.debug('Searching for intervals at 68(95)% C.L.')
tmp_list_qe_poi_dNLL = []
is_trend_changed = False
n_entries = t.GetEntriesFast()
for en in range(
1, n_entries + 1
): #add +1 to range so that to save all the entries into the segments
if en < n_entries: #if not passed the last entry
t.GetEntry(en)
# set x=quantileExpected and y=POI and create segments
qe, poi_value, dNLL = t.quantileExpected, eval(
't.{0}'.format(poi)), t.deltaNLL
#adding check of change of poi_value in case we have multidim fit.
#if the value is same as previous, we just skip.
if en == 1:
poi_value_prev = poi_value
if AreSame(poi_value, poi_value_prev):
poi_value_prev = poi_value
continue
#check if trend is change, and then change the bool is_raising which will show which vector should be filled
if (en > 2):
if ((qe < qe_prev) and is_raising) or (
(qe > qe_prev) and not is_raising):
is_trend_changed = True
self.log.debug(
'Trend of the segment has changed at entry {0}.'
.format(en))
self.log.debug(
'**********************************************'
)
#fill tmp_list_qe_poi_dNLL until we see that the trend is changed
#then put that list into self.raising_segments or self.falling_segments and clear the tmp_list_qe_poi_dNLL
if is_trend_changed or en == n_entries:
if is_raising:
self.raising_segments[poi].append(
self._get_TGraph_from_segment(
tmp_list_qe_poi_dNLL))
#self.log.debug('Appending segment to self.raising_segments[{1}]: {0}'.format(tmp_list_qe_poi_dNLL, poi))
else:
self.falling_segments[poi].append(
self._get_TGraph_from_segment(
tmp_list_qe_poi_dNLL))
#self.log.debug('Appending segment to self.falling_segments[{1}]: {0}'.format(tmp_list_qe_poi_dNLL, poi))
self.log.debug('Raising_segments state: {0}'.format(
self.raising_segments))
self.log.debug('Falling_segments state: {0}'.format(
self.falling_segments))
#delete all elements from the tmp list
del tmp_list_qe_poi_dNLL[:]
#change the state of raising/falling interval
is_raising = (not is_raising)
is_trend_changed = False
if en < n_entries:
#fill tmp_list_qe_poi_dNLL
tmp_list_qe_poi_dNLL.append((qe, poi_value, dNLL))
#self.log.debug('Entry = {2} Raising = {0}, Falling = {1}'.format(is_raising, (not is_raising), en))
#self.log.debug('Entry = {4} Filling tmp_list_qe_poi_dNLL (size={0}) with : qe = {1}, poi_value = {2}, dNLL = {3}'.format(len(tmp_list_qe_poi_dNLL),qe, poi_value, dNLL, en ))
qe_prev = qe
class LimitVsLumi(PlotPolisher,RootPlottersBase):
def __init__(self,name = "limitVsLumi_plot" ):
self.log = Logger().getLogger(self.__class__.__name__, 10)
self.name = name
#ROOT.gSystem.AddIncludePath("-I$ROOFITSYS/include/");
ROOT.gROOT.ProcessLine(".L tdrstyle.cc")
from ROOT import setTDRStyle
ROOT.setTDRStyle(True)
ROOT.gStyle.SetPalette(1)
ROOT.gStyle.SetOptStat(0)
#self.copy_to_web_dir = False
#self.webdir = ""
self.pp = pprint.PrettyPrinter(indent=4)
def setName(self, newname): self.name = newname
def makePlot(self, data):
#self.ytitle = "U.L. @95% for |k_{3}/k_{1}|"
#self.xtitle = "L (fb^{-1})"
self.c1 =ROOT.TCanvas("cc1","95% CL limits",800,800)
self.c1.cd()
ROOT.gPad.SetRightMargin(0.0085)
try:
data['content']
except KeyError:
raise KeyError, "Canvas \'content\' dictionary is not provided in config file."
try:
data['setup']
except:
print "@@@@ Canvas \'setup\' dictionary is not provided. "
self.setup_exist=False
else:
self.arrangeCanvas(self.c1, data['setup'])
self.setup_exist=True
try:
self.leg
except AttributeError:
self.leg_exist=False
else:
self.leg_exist=True
try:
data['setup']['add_text']
except KeyError:
self.add_text_exist=False
else:
self.add_text_exist=True
i=0
rat = RootAttributeTranslator()
for theGraph in data['content']:
#theGraph = data[key]
style=rat.translate_all(theGraph['style'])
#style=theGraph['style']
#x_values = array('d', self.getColumnFromTable(theGraph['table'],'lumi'))
#y_values = array('d', self.getColumnFromTable(theGraph['table'],'ul95'))
x_values = array('d', self._getColumnFromTable(theGraph['table'],'lumi'))
y_values = array('d', self._getColumnFromTable(theGraph['table'],'ul95'))
self.log.debug("(x,z) = "+str(zip(x_values,y_values)))
#theGraph['graph'] = self.getGraph(x_values,y_values, style)
gr = SimplePlotter()
theGraph['graph'] = gr.getGraph(x_values,y_values, style)
if self.setup_exist: self.arrangeAxis(theGraph['graph'],data['setup'])
if self.leg_exist:
self.leg.AddEntry(theGraph['graph'],theGraph['legend']['text'],theGraph['legend']['opt']);
try:
theGraph['draw_opt']
except KeyError:
draw_opt = "LP"
else:
draw_opt = str(theGraph['draw_opt'])
if i==0: draw_opt += "A"
else : draw_opt += "same"
i+=1
theGraph['graph'].Draw(draw_opt)
if self.leg_exist: self.leg.Draw()
if self.add_text_exist: self.add_text(data['setup']['add_text'])
#plot_name = "limitVsLumi_2D_k3k1_0_logy"
plot_name = self.name
self.save_extensions = ['png','pdf','eps']
try:
data['setup']['save_ext']
except KeyError:
self.log.info("No extensions are provided in setup. Using default: ", self.save_extensions)
else:
self.save_extensions = list(data['setup']['save_ext'])
self.save(self.c1, plot_name, self.save_extensions)
def read_table(self, file_name):
# read it
import csv
table=[]
with open(file_name, 'r') as csvfile:
reader = csv.reader(csvfile,skipinitialspace=True,delimiter=" ")
#table = [[float(e) for e in r] for r in reader]
table = [[str(e) for e in r] for r in reader]
#table = [[e for e in r] for r in reader]
return table
def getColumnFromTable(self, table_file,col_name):
col_names_list = {'lumi':0,'bf':1,'ul68':2,'ul95':3,'wf':4}
filename = table_file.replace("_",".")+".lumi.bf.ul68.ul95.wf.tab"
self.log.debug("Reading column {0}={1} from file = {2} ...".format(col_name,col_names_list[col_name],filename))
table = self.read_table(filename)
assert len(table[0]) == len(col_names_list), "Wrong number of columns in table: %(filename)s " %locals()
#print table
column = [row[col_names_list[col_name]] for row in table]
self.log.debug("Values {0} = {1}".format(col_name,column))
return column
def _getColumnFromTable(self, table_file,col_name):
filename = table_file.replace("_",".")+".tab"
table = self.read_table(filename)
#print table
#col_names_list = {'lumi':0,'bf':1,'ul68':2,'ul95':3,'wf':4}
col_names_list = dict(zip(table[0],[i for i in range(len(table[0]))])) #e.g. {'lumi':0,'bf':1,'ul68':2,'ul95':3,'wf':4}
self.log.debug("Reading column {0}={1} from file = {2} ...".format(col_name,col_names_list[col_name],filename))
for irow in range(len(table))[1:] :
assert len(table[0]) == len(table[irow]), "In table {0}, wrong number of columns in row {1}. Should be {2} ".format(filename, len(table[irow]), len(table[0]))
#print table
column = [float(row[col_names_list[col_name.lower()]]) for row in table[1:]]
self.log.debug("Values {0} = {1}".format(col_name,column))
return column
def __init__(self):
self.my_logger = Logger()
self.log = self.my_logger.getLogger(self.__class__.__name__, 10)
self.DEBUG = self.my_logger.is_debug()
self.pp = pprint.PrettyPrinter(indent=4)
class RootHelperBase(object):
"""
Class that helps to pick any object from any root file
by specifiying the path to the object like:
path/to/root_file.root/path/to/root_object.
Class can also check for type of object and return TypeError
in case the object is not of a desired type.
"""
def __init__(self):
self.my_logger = Logger()
self.log = self.my_logger.getLogger(self.__class__.__name__, 10)
self.DEBUG = self.my_logger.is_debug()
self.pp = pprint.PrettyPrinter(indent=4)
def check_in_opened_files_table(self, file_name,access):
"""
Makes a check in the open_files_table dictionary and returns
pointer to the file if file is opened correct access mode.
Returns:
--------
tuple (is_opened, pointer_to_file).
In case file doesn't exist (is_opened=False,pointer_to_file=None)
"""
try:
self.open_files_table[file_name][access]
except KeyError:
self.log.debug('File {0} is not opened in {1} mode.'.format(file_name, access))
return (False, None)
else:
the_file = self.open_files_table[file_name][access]
if isinstance(the_file,TFile):
if (the_file.IsOpen() and not the_file.IsZombie()):
self.log.debug('File {0} is already opened in {1} mode. Return pointer to file.'.format(file_name, access))
return (True, the_file)
else:
self.log.debug('File {0} in {1} mode is either closed or zombie.'.format(file_name, access))
self.open_files_table[file_name][access] = None
return (False, None)
else:
self.log.debug('File {0} is not opened in {1} mode.'.format(file_name, access))
return (False, None)
def update_opened_files_table(self, file_object, access):
"""
Update the status of files opened.
file_name: acces : file_pointer structure.
"""
try:
self.open_files_table
except AttributeError:
#self.open_files_table = collections.OrderedDict()
self.open_files_table = {}
self.open_files_table[file_object.GetName()] = {access : file_object}
else:
try:
self.open_files_table[file_object.GetName()]
except KeyError:
self.open_files_table[file_object.GetName()] = {access : file_object}
else:
self.open_files_table[file_object.GetName()].update({access : file_object})
#self.open_files_table['dummy'].update({access : file_object})
#self.open_files_table[file_object.GetName()][access] = file_object
if self.DEBUG:
self.pp.pprint(self.open_files_table)
return 0
def TFile_safe_open(self, file_name, access = 'READ'):
"""
Safely open TFile object. Memory is saved by cheking if the file is already
open by looking up in the list open_files_table.
"""
#check if file is already openedby looking-up the opend files dict
is_opened=False
rootfile= None
try:
self.open_files_table
except AttributeError:
pass
else:
is_opened, rootfile = self.check_in_opened_files_table(file_name,access)
if is_opened:
self.log.debug('Returning pointer to ROOT file: {0}'.format(file_name))
return rootfile
self.log.debug('Opening ROOT file: {0}'.format(file_name))
if access.upper() == 'READ' and not os.path.exists(file_name):
raise IOError, 'File path does not exist: {0}'.format(file_name)
else:
base_dir = os.path.dirname(file_name)
misc.make_sure_path_exists(base_dir)
rootfile = TFile.Open(file_name,access)
self.update_opened_files_table(rootfile, access)
if not rootfile:
raise IOError, 'The file {0} either doesn\'t exist or cannot be open'.format(file_name)
return rootfile
def get_paths(self, path):
"""
Returns tuple (path_to_root_file, path_to_root_object_in_root_file)
"""
path_contains_file = ('.root' in path)
path_segments = path.split('.root')
if path.endswith('.root'): #only root file path exists
return (path,"")
#print path_segments
#assert 1<len(path_segments)<=2, 'Path should be in format <path/to/dir>root_object_file.root/path_to_root_object_in_file'
assert 0<len(path_segments)<=2, 'Path should be in format <path/to/dir>root_object_file.root/path_to_root_object_in_file'
path_to_file = ""
if len(path_segments)==2: #path contains file name and object path in the root file
path_to_file = path_segments[0]+'.root'
self.log.debug('Src root file: {0}'.format(path_to_file ))
#path_to_root_object = string.join(path_segments[-1].split('/')[1:],'/') #to remove the '/' after .root
if path_segments[-1].startswith('/'):
path_to_root_object = path_segments[-1][1:] #to remove the '/' after .root
else:
path_to_root_object = path_segments[-1] #there is no '/' at the beggining
self.log.debug('Src root_object name: {0}'.format(path_to_root_object))
return (path_to_file,path_to_root_object)
#path_to_file = path_segments[0]+'.root'
#self.log.debug('Src root file: {0}'.format(path_to_file ))
#path_to_root_object = string.join(path_segments[-1].split('/')[1:],'/') #to remove the '/' after .root
#self.log.debug('Src root_object name: {0}'.format(path_to_root_object))
#return (path_to_file,path_to_root_object)
def get_object(self, path, object_type=None, clone=False):
"""
Get any root object copy from path and check it's type.
The object is copied from the file if needed.
"""
path_to_file, path_to_root_object = self.get_paths(path)
root_object_file = self.TFile_safe_open(path_to_file, 'READ')
the_object = root_object_file.Get(path_to_root_object)
is_TTree = isinstance(the_object,TTree)
if clone:
if not is_TTree:
the_object = copy.deepcopy(root_object_file.Get(path_to_root_object))
self.log.debug('Coping root_object {0} of type={1}.'.format(path_to_root_object, type(the_object)))
root_object_file.Close()
else:
#FIXME
self.log.warn('Cloning the full tree {0}. !!! Still not fully tested !!!'.format(path_to_root_object))
the_object = root_object_file.Get(path_to_root_object).CloneTree()
#will not close file since it will destroy the object. Better to write the tree down first, then close file.
else:
self.log.debug('Pointer to root_object {0} of type={1} is returned.'.format(path_to_root_object, type(the_object)))
return the_object
def get_TTree(self,path , cut = None, clone = False):
"""
Get a tree from the path of format //machine/file_name.root/subdir/tree_name.
If path is list it will asume TChain. Wildcards can be used but ".root" has
to exost in the path name, otherwise 'segmentation violation'
"""
the_tree = TChain()
if isinstance(path, list):
tree_name = self.get_paths(path[0])[1]
the_tree.SetName(tree_name)
for item in path:
assert isinstance(item,str),'The tree path should be of string format and not: {0}'.format(type(item))
add_result = the_tree.Add(item)
elif isinstance(path, str):
tree_name = self.get_paths(path)[1]
the_tree.SetName(tree_name)
add_result = the_tree.Add(path)
self.log.debug('TChain has been constructed from {0} files with correct tree names.'.format(add_result))
if cut:
assert isinstance(cut, str), 'The TTree cut has to be string value, not {0} !!!'.format(type(cut))
clone = True
the_selection_tree = the_tree.CopyTree(cut)
return the_selection_tree
else:
return the_tree
def get_histogram(self,path, hist_type = TH1, clone = False):
"""
Get TH1 object or any other that inherits from TH1
"""
return self.get_object(path,hist_type, clone)
def get_embedded_object(self, path_to_container, container_type = None, embedded_object = None, object_type = None, clone = False):
"""
Get an object embedded into another class, like e.g. a TH1 from TCanvas
saved in file. In case only path_to_container is given, it will return the container
like with get_object method.
"""
pass
def add_to_basket(self,root_object, new_name = None, new_title = None):
"""
Add object to the basket with new_name and new_title.
If new_name contains "/" then a directory will be created inside the file. (TODO)
"""
if new_name:
#name_in_basket = new_name
new_name_no_subfolders = new_name.split('/')[-1] #remove subfolder name from the new_name
root_object.SetName(new_name_no_subfolders)
name_in_basket = new_name
else:
name_in_basket = root_object.GetName()
if new_title:
root_object.SetTitle(new_title)
try:
self.root_fruit_basket
except AttributeError:
self.root_fruit_basket = collections.OrderedDict()
self.log.debug('Creating new root-object basket.')
else:
if self.DEBUG and len(self.root_fruit_basket)<10:
self.log.debug('Adding root-object to existing basket. Basket state (printed if less then 10 items):')
self.pp.pprint(self.root_fruit_basket)
self.root_fruit_basket[name_in_basket] = root_object
def _get_subfolders_and_name(self,path):
"""
Gives back the 2 element tuple with subfolder path and a name of root_object
"""
path_segments = path.split('/')
assert len(path_segments)>0, 'The name should not be empty string.'
if len(path_segments) > 1:
#check if first is '/'
if path_segments[0]=='': path_segments.pop(0)
subfolders = string.join(path_segments[:-1],'/')
root_object_name = path_segments[-1]
self.log.debug('Root-subfolder: {0}'.format(subfolders))
self.log.debug('Root-object name: {0}'.format(root_object_name))
return (subfolders, root_object_name)
else:
root_object_name = path_segments[-1]
return (None, root_object_name)
def _get_directory(self,root_file, path):
"""
Create and cd to the directory if given like a/b/c
"""
root_file.cd()
#subfolders = self._get_subfolders_and_name(path)[0]
if path:
self.log.debug('Creating root-subfolder {0}'.format(path))
mkdir_res = root_file.mkdir(path)
self.log.info('Root-subfolder {0} created with code = {1}'.format(path, mkdir_res))
root_file.cd(path)
else: #no subfolder will be created
root_file.cd()
self.log.debug('Current directory: {0}'.format(gDirectory.GetPath()))
def flush_basket(self):
"""
Resets the basket content and delets the basket.
"""
try:
del self.root_fruit_basket
except:
raise RuntimeError, 'Basket cannot be flushed and destroyed! It even doesn\'t exist ...'
else:
self.log.info('Basket flushed!')
return 0
def dump_basket_to_file(self, file_name, access = 'UPDATE'):
"""
Save what is in basket to a file. Create directories in the path if needed.
"""
out_file = self.TFile_safe_open(file_name, access)
out_file.cd()
if self.DEBUG:
self.log.debug('Dumping basket to file.')
self.pp.pprint(self.root_fruit_basket)
for item_name in self.root_fruit_basket.keys():
subfolders, root_object_name = self._get_subfolders_and_name(item_name)
self._get_directory(out_file, subfolders) #it will create and cd to the directory if given like a/b/c
self.log.debug('Writing root-object: {0} Object name: {1} ; Object title: {2}'.format(self.root_fruit_basket[item_name],self.root_fruit_basket[item_name].GetName(),self.root_fruit_basket[item_name].GetTitle()))
is_TTree = isinstance(self.root_fruit_basket[item_name],TTree)
if is_TTree:
self.log.debug('This is a TTree object : {0}'.format(self.root_fruit_basket[item_name]))
copy_tree_name = self.root_fruit_basket[item_name].GetName()
copy_tree_title = self.root_fruit_basket[item_name].GetTitle()
tree_for_saving = self.root_fruit_basket[item_name].CloneTree(0)
copy_res = tree_for_saving.CopyEntries(self.root_fruit_basket[item_name])
tree_for_saving.SetNameTitle(copy_tree_name,copy_tree_title)
write_res = tree_for_saving.Write()
else:
write_res = self.root_fruit_basket[item_name].Write()
if write_res == 0 :
self.log.error('The object {0} cannot be written into {1}'.format(item_name, gDirectory.GetPath()))
else:
self.log.info('The object {0} has been written into {1}'.format(item_name, gDirectory.GetPath()))
out_file.Close()
self.log.info('Saved the basket with {1} items into the file: {0}'.format(file_name, len(self.root_fruit_basket)))
self.flush_basket()
return 0