def treatFile ( self, the_file ) :
"""Add the file to TChain
"""
## suppress Warning/Error messages from ROOT
from ostap.logger.utils import rootError
with rootError() :
## new temporary chani/tree
tree = ROOT.TChain ( self.chain_name )
tree.Add ( the_file )
ok = len ( tree )
if ok :
if self.check and self.files :
chain = self.chain
self.check_trees ( chain , tree , the_file )
del chain
Files.treatFile ( self , the_file )
else :
self.__bad_files.add ( the_file )
if not self.silent :
logger.warning ( "No/empty chain '%s' in file '%s'" % ( self.chain_name , the_file ) )
del tree
def getLumi ( self ):
"""Get the luminosity
"""
## suppress Warning/Error messages from ROOT
from ostap.logger.utils import rootError
with rootError() :
from ostap.contrib.lhcb.lumi import getLumi
return getLumi ( self.chain2 )
def _make_dataset_(tree,
variables,
selection='',
name='',
title='',
silent=False):
"""Create the dataset from the tree
>>> tree = ...
>>> ds = tree.make_dataset ( [ 'px , 'py' , 'pz' ] )
"""
import ostap.trees.cuts
import ostap.fitting.roofit
cuts = ROOT.TCut(selection)
varset = ROOT.RooArgSet()
for v in variables:
if isinstance(v, str): v = Variable(v)
elif isinstance(v, (tuple, list)): v = Variable(*v)
elif isinstance(v, dict): v = Variable(**v)
assert isinstance(
v, Variable), "Can't create Variable from %s/%s" % (v, type(v))
assert v.trivial, "Variable %s is not ``trivial''" % v.name
assert hasattr(tree,
v.name), "Tree/Chain has no branch ``%s''" % v.name
varset.add(v.var)
mn, mx = v.minmax
if _minv < mn: cuts &= "%.16g <= %s" % (mn, v.name)
if _maxv > mx: cuts &= "%s <= %.16g" % (v.name, mx)
if not name:
from ostap.core.core import dsID
name = '%s_%s' % (dsID(), tree.GetName())
if not title: title = '%s/%s' % (name, tree.GetTitle())
total = len(tree)
processed = tree.statVar('1', selection).nEntries()
skipped = tree.statVar('1', str(cuts)).nEntries()
stat = total, processed, processed - skipped
from ostap.logger.utils import rooSilent, rootError
with rooSilent(ROOT.RooFit.ERROR, True):
with rootError(ROOT.kWarning):
ds = ROOT.RooDataSet(name, title, tree, varset, str(cuts))
if not silent:
from ostap.logger.logger import attention
skipped = 'Skipped:%d' % stat[2]
skipped = '/' + attention(skipped) if stat[2] else ''
logger.info(
'make_dataset: Events Total:%d/Processed:%s%s CUTS: "%s"\n# %s' %
(stat[0], stat[1], skipped, selection, ds))
return ds, stat
def getLumi ( self ):
"""Get the luminosity
"""
## suppress Warning/Error messages from ROOT
with rootError() :
try :
return getLumi ( self.chain2 )
except ImportError :
logger.error('DataAndLumi:getLumi is not available!')
return -1.e+6
def treatFile ( self, the_file ) :
"""Add the file to TChain
"""
## suppress Warning/Error messages from ROOT
from ostap.logger.utils import rootError
with rootError() :
tree1 = ROOT.TChain ( self.chain1_name )
tree1.Add ( the_file )
tree2 = ROOT.TChain ( self.chain2_name )
tree2.Add ( the_file )
ok1 = len ( tree1 )
if self.check and self.files1 and ok1 :
chain1 = self.chain1
self.check_trees ( tree1 , chain1 , the_file )
del chain1
ok2 = len ( tree2 )
if self.check and self.files2 and ok2 :
chain2 = self.chain2
self.check_trees ( tree2 , chain2 , the_file )
del chain2
if ok1 and ok2 :
Files.treatFile ( self , the_file )
elif ok2 :
self.bad_files1.add ( the_file )
if not self.silent :
logger.warning ( "No/empty chain1 '%s' in file '%s'" % ( self.chain1_name , the_file ) )
elif ok1 :
self.bad_files2.add ( the_file )
if not self.silent :
logger.warning ( "No/empty chain2 '%s' in file '%s'" % ( self.chain2_name , the_file ) )
else :
self.bad_files1.add ( the_file )
self.bad_files2.add ( the_file )
if not self.silent :
logger.warning ( "No/empty chains '%s'/'%s' in file '%s'" % ( self.chain1_name ,
self.chain2_name , the_file ) )
del tree1
del tree2
def treatFile(self, the_file):
"""Add the file to TChain
"""
Files.treatFile(self, the_file)
## suppress Warning/Error messages from ROOT
from ostap.logger.utils import rootError
with rootError():
tmp = ROOT.TChain(self.chain.GetName())
tmp.Add(the_file)
##if 0 < tmp.GetEntries() and 0 < tmp.GetEntry(0) : self.chain.Add ( the_file )
if 0 < tmp.GetEntries(): self.chain.Add(the_file)
else:
logger.warning("No/empty chain '%s' in file '%s'" %
(self.chain.GetName(), the_file))
self.e_list1.add(the_file)
def treatFile(self, the_file):
"""Add the file to TChain
"""
## suppress Warning/Error messages from ROOT
from ostap.logger.utils import rootError
with rootError():
tmp1 = ROOT.TChain(self.chain.GetName())
tmp1.Add(the_file)
if tmp1:
Files.treatFile(self, the_file)
self.chain.Add(the_file)
else:
self.e_list1.add(the_file)
if not self.silent:
logger.warning("No/empty chain '%s' in file '%s'" %
(self.chain.GetName(), the_file))
def treatFile(self, the_file):
"""Add the file to TChain
"""
## suppress Warning/Error messages from ROOT
from ostap.logger.utils import rootError
with rootError():
tmp1 = ROOT.TChain(self.chain.GetName())
tmp1.Add(the_file)
tmp2 = ROOT.TChain(self.chain2.GetName())
tmp2.Add(the_file)
if tmp1: self.check_trees(tmp1, self.chain, the_file)
if tmp2: self.check_trees(tmp2, self.chain2, the_file)
if tmp1 and tmp2:
Files.treatFile(self, the_file)
self.chain.Add(the_file)
self.chain2.Add(the_file)
self.__files2.append(the_file)
elif tmp2 and not tmp1 and self.missing1st:
self.e_list1.add(the_file)
if not self.silent:
logger.warning("No/empty chain1 '%s' in file '%s'" %
(self.chain.GetName(), the_file))
self.chain2.Add(the_file)
self.__files2.append(the_file)
elif tmp1 and not tmp2 and self.missing2nd:
self.e_list2.add(the_file)
if not self.silent:
logger.warning("No/empty chain2 '%s' in file '%s'" %
(self.chain2.GetName(), the_file))
self.chain.Add(the_file)
self.set_files(self.files + (the_file, ))
else:
self.e_list1.add(the_file)
self.e_list2.add(the_file)
if not self.silent:
logger.warning("No/empty chains '%s'/'%s' in file '%s'" %
(self.chain.GetName(),
self.chain2.GetName(), the_file))
def _load_macro_ ( macro , silent = True ) :
"""Load ROOT macro"""
logger.debug ("Try to load macro '%s'" % macro )
if silent :
from ostap.logger.utils import rootError
with rootError() : sc = ROOT.gROOT.LoadMacro ( macro )
else :
sc = ROOT.gROOT.LoadMacro ( macro )
if sc :
# - Interactive mode: print traceback and continue
if arguments.batch :
raise RuntimeError ( 'Failure to load macro "%s" code:%d' % ( macro , sc ) )
else :
logger.error ('Failure to load macro "%s" code:%d' % ( macro , sc ) )
return False
logger.debug ("Loaded macro '%s'" % macro )
del sc
return True
def valid_formula ( expression , varset ) :
expression = expression.strip()
if not expression : return True
if isinstance ( varset , ROOT.RooArgSet ) :
vlst = ROOT.RooArgList()
for v in varset : vlst.add ( v )
result = valid_formula ( expression , vlst )
del vlst
return result
assert isinstance ( varset , ROOT.RooArgList ), 'Invalid type %s' % type (varset)
from ostap.logger.utils import rooSilent, rootError
with rooSilent ( ROOT.RooFit.FATAL + 1 , True ) :
with rootError( ROOT.kError + 1 ) :
_f = ROOT.RooFormulaVar( '' , expression , varset )
fok = _f.ok ()
del _f
return fok
def make_dataset ( tree , variables , selection = '' , name = '' , title = '' , silent = False ) :
"""Create the dataset from the tree
>>> tree = ...
>>> ds = tree.make_dataset ( [ 'px , 'py' , 'pz' ] )
"""
import ostap.trees.cuts
import ostap.fitting.roofit
varset = ROOT.RooArgSet()
vars = set()
formulas = []
selection = str ( selection ) if isinstance ( selection , ROOT.TCut ) else selection
selection = selection.strip() if isinstance ( selection , str ) else selection
cuts = [ selection ] if selection else []
for v in variables :
if isinstance ( v , str ) : vv = Variable ( v )
elif isinstance ( v , ROOT.RooRealVar ) : vv = Variable ( v )
elif isinstance ( v , ( tuple , list ) ) : vv = Variable ( *v )
elif isinstance ( v , dict ) : vv = Variable ( **v )
elif isinstance ( v , Variable ) : vv = v
else :
logger.error("Do not know how to treat the variable %s/%s, skip it" % ( v , type ( v ) ) )
continue
if vv.trivial and vv.name == vv.formula :
assert hasattr ( tree , vv.name ) , "Tree/Chain has no branch ``%s''" % vv.name
assert hasattr ( tree , vv.name ) , "Tree/Chain has no branch ``%s''" % vv.name
varset.add ( vv.var )
vars.add ( vv )
elif vv.formula :
formulas.append ( vv )
continue
else :
logger.error("Do not know how to treat the variable %s, skip it" % vv.name )
continue
mn , mx = vv.minmax
if _minv < mn : cuts.append ( "(%.16g <= %s)" % ( mn , vv.name ) )
if _maxv > mx : cuts.append ( "(%s <= %.16g)" % ( vv.name , mx ) )
##
cuts = ROOT.TCut(' && '.join(cuts) ) if cuts else ROOT.TCut()
## extended varset
stor = set()
varsete = ROOT.RooArgSet()
for v in varset : varsete.add ( v )
expressions = [ f.formula for f in formulas ]
if selection : expressions.append ( selection )
if expressions :
tt = None
if isinstance ( tree , ROOT.TChain ) :
nf = len ( tree.files() )
for i in range ( nf ) :
tt = tree[i]
if tt : break
if not tt : tt = tree
from ostap.core.core import fID
for expression in expressions :
tf = Ostap.Formula ( fID () , str ( expression ) , tt )
assert tf.ok() , 'Invalid formula %s' % expression
i = 0
leaf = tf.GetLeaf( i )
while leaf :
lname = leaf.GetName()
if not lname in varsete :
v = Variable ( lname )
varsete.add ( v.var )
stor.add ( v )
i += 1
leaf = tf.GetLeaf( i )
del tf
if not name :
from ostap.core.core import dsID
name = '%s_%s' % ( dsID() , tree.GetName() )
if not title : title = '%s/%s' % ( name , tree.GetTitle() )
total = len ( tree )
processed = tree.statVar ( '1' , selection ).nEntries()
skipped = tree.statVar ( '1' , str ( cuts ) ).nEntries()
stat = total, processed , processed - skipped
from ostap.logger.utils import rooSilent, rootError
with rooSilent ( ROOT.RooFit.ERROR , True ) :
with rootError( ROOT.kWarning ) :
ds = ROOT.RooDataSet ( name , title , tree , varsete , str( cuts ) )
varsete = ds.get()
## add complex expressions
if formulas :
# a
vset = ds.get()
vlst = ROOT.RooArgList()
for v in vset : vlst.add ( v )
fcols = ROOT.RooArgList()
ffs = []
fcuts = []
for f in formulas :
fv = ROOT.RooFormulaVar ( f.name , f.description , f.formula , vlst )
assert fv.ok() , 'Invalid formula: %s' % f.formula
ffs.append ( fv )
fcols.add ( fv )
mn , mx = f.minmax
if _minv < mn : fcuts.append ( "(%.16g <= %s)" % ( mn , fv.name ) )
if _maxv > mx : fcuts.append ( "(%s <= %.16g)" % ( fv.name , mx ) )
ds.addColumns ( fcols )
## apply cuts (if any) for the complex expressions
if fcuts :
fcuts = [ '(%s)' % f for f in fcuts ]
fcuts = ' && '.join ( fcuts )
_vars = ds.get()
ds1 = ROOT.RooDataSet ( dsID() , ds.title , ds , _vars , fcuts )
ds.clear()
del ds
ds = ds1
varsete = ds.get()
nvars = ROOT.RooArgSet()
for v in varset : nvars.add ( v )
for v in formulas : nvars.add ( v.var )
varset = nvars
varsete = ds.get()
## remove all temporary variables
if len ( varset ) != len ( varsete ) :
ds1 = ds.reduce ( varset )
ds.clear()
del ds
ds = ds1
if not silent :
skipped = 'Skipped:%d' % stat[2]
skipped = '/' + attention ( skipped ) if stat[2] else ''
logger.info (
'make_dataset: Events Total:%d/Processed:%s%s CUTS: "%s"\n# %s' % (
stat[0] ,
stat[1] ,
skipped ,
selection , ds ) )
return ds , stat
def getLumi(data, *args):
"""Get lumi :
>>> l1 = getLumi ( 'myfile.root' )
>>> l2 = getLumi ( tree )
>>> l3 = getLumi ( chain )
>>> l4 = getLumi ( file )
>>> l5 = getLumi ( [ any sequence of above ] )
"""
#
if args:
data = [data]
for a in args:
data.append(a)
return getLumi(data)
if isinstance(data, str):
## expand the actual file name
data = os.path.expandvars(data)
data = os.path.expanduser(data)
data = os.path.expandvars(data)
data = os.path.expandvars(data)
try:
tree = ROOT.TChain(lumi_tree)
tree.Add(data)
return getLumi(tree)
except:
logger.error('Unable to get lumi/1 for %s' % data)
return VE()
#
if isinstance(data, ROOT.TFile):
try:
tree = data.Get(lumi_tree)
return getLumi(tree)
except:
logger.error('Unable to get lumi/2 for %s' % data.GetName())
return VE()
if isinstance(data, ROOT.TTree):
## try :
with rootError(): ## suppress errors from ROOT
## if hasattr ( data , 'pstatVar' ) :
## stat = data.pstatVar ( [ lumi , lumi_err ] , lumi_cuts , chunk_size = -1 , max_files = 10 )
## else :
stat = data.statVar([lumi, lumi_err], lumi_cuts)
##
s1 = stat[lumi]
s2 = stat[lumi_err]
##
return VE(s1.sum(), s2.sum()**2)
##except :
## logger.error('Unable to get lumi/3 for %s' % data.GetName() )
## return VE()
l = VE()
for i in data:
k = getLumi(i)
## @attention: linear addition of uncertainties:
l = VE(l.value() + k.value(), (l.error() + k.error())**2)
return l
def getLumi ( data , *args ) :
"""Get lumi :
>>> l1 = getLumi ( 'myfile.root' )
>>> l2 = getLumi ( tree )
>>> l3 = getLumi ( chain )
>>> l4 = getLumi ( file )
>>> l5 = getLumi ( [ any sequence of above ] )
"""
tree_name = 'GetIntegratedLuminosity/LumiTuple'
#
from ostap.core.core import VE, hID
import ostap.io.root_file
import ostap.trees.trees
#
if args :
data = [ data ]
for a in args : data.append ( a )
return getLumi ( data )
if isinstance ( data , str ) :
## expand the actual file name
import os
data = os.path.expandvars ( data )
data = os.path.expanduser ( data )
data = os.path.expandvars ( data )
data = os.path.expandvars ( data )
try :
tree = ROOT.TChain ( tree_name )
tree.Add ( data )
lumi = getLumi ( tree )
return lumi
except :
logger.error('Unable to get lumi(1) for %s' % data )
return VE()
#
if isinstance ( data , ROOT.TFile ) :
try :
tree = data.Get( tree_name )
return getLumi ( tree )
except:
logger.error('Unable to get lumi(2) for %s' % data.GetName() )
return VE()
if isinstance ( data , ROOT.TTree ) :
## print data
from ostap.logger.utils import rootError
try:
#
with rootError() : ## suppress errors from ROOT
## @attention here we are using sumVar!
l1 = data.sumVar ( '1.0*IntegratedLuminosity+0.0*IntegratedLuminosityErr' , '0<=IntegratedLuminosity' )
l2 = data.sumVar ( '1.0*IntegratedLuminosity+1.0*IntegratedLuminosityErr' , '0<=IntegratedLuminosity' )
l3 = data.sumVar ( '1.0*IntegratedLuminosity-1.0*IntegratedLuminosityErr' , '0<=IntegratedLuminosity' )
#
l1.setError ( 0.5 * abs ( l2.value () - l3.value () ) )
#
l0 = data.sumVar ( 'IntegratedLuminosity' , '0 >IntegratedLuminosity' )
if 0 != l0.value() : logger.error( 'Something weird happens with Lumi/1: %s' % l0 )
l0 = data.sumVar ( 'IntegratedLuminosity' , 'IntegratedLuminosity>100000' )
if 0 != l0.value() : logger.error( 'Something weird happens with Lumi/2: %s' % l0 )
l0 = data.sumVar ( 'IntegratedLuminosity' , '0>IntegratedLuminosityErr' )
if 0 != l0.value() : logger.error( 'Something weird happens with Lumi/3: %s' % l0 )
#
return l1
except :
logger.error('Unable to get lumi(3) for %s' % data.GetName() )
return VE()
l = VE()
for i in data :
k = getLumi ( i )
## @attention: linear addition of uncertainties:
l = VE ( l.value() + k.value() , ( l.error() + k.error () ) ** 2 )
return l
def getLumi(data, *args):
"""Get lumi :
>>> l1 = getLumi ( 'myfile.root' )
>>> l2 = getLumi ( tree )
>>> l3 = getLumi ( chain )
>>> l4 = getLumi ( file )
>>> l5 = getLumi ( [ any sequence of above ] )
"""
#
if args:
data = [data]
for a in args:
data.append(a)
return getLumi(data)
if isinstance(data, str):
## expand the actual file name
data = os.path.expandvars(data)
data = os.path.expanduser(data)
data = os.path.expandvars(data)
data = os.path.expandvars(data)
try:
tree = ROOT.TChain(lumi_tree)
tree.Add(data)
return getLumi(tree)
except:
logger.error('Unable to get lumi/1 for %s' % data)
return VE()
#
if isinstance(data, ROOT.TFile):
try:
tree = data.Get(lumi_tree)
return getLumi(tree)
except:
logger.error('Unable to get lumi/2 for %s' % data.GetName())
return VE()
if isinstance(data, ROOT.TTree):
## try :
with rootError(): ## suppress errors from ROOT
## if ( 6 , 25 ) < root_info :
## ## use DataFrames. It is faster here? not sure....
## from ostap.frame.frames import DataFrame , frame_statVars
## frame = DataFrame ( data )
## stat = frame_statVars ( frame , [ lumi , lumi_err ] , lumi_cuts )
## else :
stat = data.statVar([lumi, lumi_err], lumi_cuts)
##
s1 = stat[lumi]
s2 = stat[lumi_err]
##
return VE(s1.sum(), s2.sum()**2)
##except :
## logger.error('Unable to get lumi/3 for %s' % data.GetName() )
## return VE()
l = VE()
for i in data:
k = getLumi(i)
## @attention: linear addition of uncertainties:
l = VE(l.value() + k.value(), (l.error() + k.error())**2)
return l
