def frame(self):
"""Get the DataFrame for the chain"""
from ostap.frames.frames import DataFrame
f = DataFrame(self.chain)
if not self.filent:
pb = f.ProgressBar(len(self.chain))
return f
def fill_tree(tname, fname):
tdf = DataFrame(1000)
a = tdf.ProgressBar(1000)
tdf.Define ("one", "1.0" )\
.Define ("b1" , "(double) tdfentry_")\
.Define ("b2" , "(1.0+b1)*(1.0+b1)" )\
.Snapshot ( tname, fname )
def frame2 ( self ) :
"""``frame2'': Get the DataFrame for the second chain"""
from ostap.frames.frames import DataFrame, frame_progress
f = DataFrame ( self.chain2 )
if not self.silent :
pb = frame_progres ( f , len ( self.chain2 ) )
return f
def frame ( self ) :
"""Get the DataFrame for the chain"""
from ostap.frames.frames import DataFrame, frame_progress
f = DataFrame ( self.chain )
if not self.filent:
pb = frame_progress ( f , len ( self.chain ) )
return f
def test_frame3 () :
logger = getLogger ( 'test_frame2' )
if root_info < ( 6 , 16 ) :
logger.warning ( "Test is disabled for this version of ROOT %s" % str ( root_info ) )
return
tree = Tree ( name = tname , file = fname ).chain
for i in range ( 5 ) :
frame = DataFrame ( tname , fname )
pb = frame_progress ( frame , len ( tree ) )
logger.info ( 'Value: %d %s' % ( i , pb.GetValue() ) )
def test_frame2 ( ) :
logger = getLogger ( 'test_frame2' )
if root_info < ( 6 , 16 ) :
logger.warning ( "Test is disabled for this version of ROOT %s" % str ( root_info ) )
return
frame = DataFrame ( tname , fname )
tree = Tree ( name = tname , file = fname ).chain
s1 = tree.statVar ( 'b1' , '1/b1' )
s2 = frame_statVar ( tree , 'b1' , '1/b1' )
logger.info ('StatTree :%s' % s1 )
logger.info ('StatFrame :%s' % s2 )
def test_frame1 ( ) :
logger = getLogger ( 'test_frame1' )
if root_info < ( 6 , 16 ) :
logger.warning ( "Test is disabled for this version of ROOT %s" % str ( root_info ) )
return
frame = DataFrame ( tname , fname )
tree = Tree ( name = tname , file = fname ).chain
h1 = tree .draw ( 'b1' , '1/b1' )
h2 = frame.draw ( 'b1' , '1/b1' )
h1 = ROOT.TH1D( hID() , '' , 100 , 0 , 1000 )
h2 = ROOT.TH1D( hID() , '' , 100 , 0 , 1000 )
tree.project ( h1 , 'b1' , '1.0/b1' )
frame_project ( tree , h2 , 'b1' , '2.0/b1' )
with wait ( 3 ), use_canvas ( 'test_frame1' ) :
h1.red ()
h2.blue ()
h1.draw ()
h2.draw ( 'same hist' )
# A simple helper function to fill a test tree
def fill_tree(tname, fname):
tdf = DataFrame(1000)
a = tdf.ProgressBar(1000)
tdf.Define ("one", "1.0" )\
.Define ("b1" , "(double) tdfentry_")\
.Define ("b2" , "(1.0+b1)*(1.0+b1)" )\
.Snapshot ( tname, fname )
# We prepare an input tree to run on
tname = "myTree"
fill_tree(tname, fname)
frame = DataFrame(tname, fname)
tree = Tree(name=tname, file=fname).chain
from ostap.utils.utils import implicitMT
from ostap.utils.timing import timing
def test_frame0():
for mt in (False, True):
with implicitMT(mt):
logger.info(80 * '*')
logger.info('MT enabled? %s/%s' %
(ROOT.ROOT.IsImplicitMTEnabled(),
def __init__(
self,
chain, ## input TChain/TTree
selection={}, ## selection/cuts
save_vars=(), ## list of variables to save
new_vars={}, ## new variables
no_vars=(), ## exclude these variables
##
output='', ## output file name
name='', ## the name
addselvars=False, ## add varibles from selections?
tmp_keep=False, ## keep the temporary file
silent=False): ## silent processing
from ostap.frames.frames import DataFrame
frame = DataFrame(chain)
report = None
if not silent:
pbar = frame.ProgressBar(len(chain))
nvars = []
## new variables
for nv in new_vars:
frame = frame.Define(nv, new_vars[nv])
nvars.append(nv)
from ostap.core.ostap_types import (string_types, listlike_types,
dictlike_types)
cut_types = string_types + (ROOT.TCut, )
Lmax = 30
selections = []
if selection and isinstance(selection, cut_types):
ss = str(selection).strip()
if len(ss) < Lmax: filter_name = ss
else: filter_name = 'SELECTION'
frame = frame.Filter(ss, filter_name)
selections.append(ss)
elif selection and isinstance(selection, dictlike_types):
for filter_name in selection:
s = selection[filter_name]
assert isinstance ( s , cut_types ),\
'Invalid selection type %s/%s' % ( s , type ( s ) )
ss = str(s).strip()
frame = frame.Filter(ss, str(filter_name))
selections.append(ss)
elif selection and isinstance(selection, listlike_types):
for i, s in enumerate(selection):
assert isinstance ( s , cut_types ),\
'Invalid selection type %s/%s' % ( s , type ( s ) )
ss = str(s).strip()
##
if len(ss) < Lmax: filter_name = ss
else: filter_name = 'SELECTION%d' % i
#
frame = frame.Filter(ss, filter_name)
selections.append(ss)
elif selection:
raise TypeError('Invalid selection type %s/%s' %
(selection, type(selection)))
if not output:
output = self.tempfile(prefix='ostap-frame-', suffix='.root')
## logger.debug ( 'ReduceTree: output file is %s' % output )
if not tmp_keep: self.trash.add(output)
if selections: report = frame.Report()
if selections and addselvars:
bvars = chain.the_variables(selections)
save_vars = list(bvars) + [v for v in save_vars if not v in bvars]
save_vars = tuple(save_vars)
## exclude some variables
if no_vars and not save_vars:
bvars = list(chain.branches())
all_vars = list(bvars) + [v for v in nvars if not v in bvars]
save_vars = tuple([v for v in all_vars if not v in no_vars])
elif no_vars:
bvars = chain.the_variables(*save_vars)
all_vars = list(bvars) + [v for v in nvars if not v in bvars]
save_vars = tuple([v for v in all_vars if not v in no_vars])
nb_ = len(chain.branches())
ne_ = len(chain)
## chain name:
## FIXME!
# cname = chain.GetName() ## produces ROOT error
if not name:
_, _, cname = chain.GetName().rpartition('/')
name = '%s_reduced' % cname
self.__name = name
if not save_vars:
snapshot = frame.Snapshot(name, output)
else:
bvars = chain.the_variables(*save_vars)
all_vars = list(bvars) + [v for v in nvars if not v in bvars]
from ostap.core.core import strings as _strings
all_vars = _strings(all_vars)
snapshot = frame.Snapshot(name, output, all_vars)
assert os.path.exists ( output ) and\
os.path.isfile ( output ) , 'Invalid file %s' % fname
self.__chain = ROOT.TChain(name)
self.__chain.Add(output)
self.__output = output
self.__report = 'Tree -> Frame -> Tree filter/transformation'
self.__table = []
if report:
from ostap.frames.frames import report_print, report_as_table
title = self.__report
self.__report += '\n%s' % report_print(report, title, '# ')
self.__table = report_as_table(report)
fs = os.path.getsize(self.__output)
gb, r = divmod(fs, 1024 * 1024 * 1024)
mb, r = divmod(r, 1024 * 1024)
kb, r = divmod(r, 1024)
if gb: fs = '%.1fGB' % (float(fs) / 1024 / 1024 / 1024)
elif mb: fs = '%.1fMB' % (float(fs) / 1024 / 1024)
elif kb: fs = '%.1fkB' % (float(fs) / 1024)
else: fs = '%sB' % fs
nb = len(self.__chain.branches())
ne = len(self.__chain)
self.__report += '\n# Reduce %d -> %d branches, %d -> %d entries' % (
nb_, nb, ne_, ne)
self.__report += '\n# Output:%s size:%s' % (self.__output, fs)
self.__report += '\n# %s' % str(self.__chain)
def test_frame0 () :
logger = getLogger ( 'test_frame0' )
if root_info < (6,16) :
logger.warning ( "Test is disabled for this version of ROOT %s" % str ( root_info ) )
return
frame = DataFrame ( tname , fname )
tree = Tree ( name = tname , file = fname ).chain
for mt in ( False , True ) :
with implicitMT ( mt ) :
logger.info ( 80*'*' )
logger.info ( 'MT enabled? %s' % ROOT.ROOT.IsImplicitMTEnabled() )
logger.info ( 'Tree :\n%s' % tree )
logger.info ( 'Frame :\n%s' % frame )
logger.info ( 'Len: %30s vs %-30s ' % ( len ( frame ) , len ( tree ) ) )
logger.info ( 'Branches: %30s vs %-30s ' % ( frame.branches() , tree.branches() ) )
logger.info ( 'nEff: %30s vs %-30s ' % ( frame.nEff () , tree.nEff () ) )
logger.info ( 'nEff(b1): %30s vs %-30s ' % ( frame.nEff ('b1') , tree.nEff ('b1') ) )
logger.info ( "m(5,50,'b1','b1/(b2+1'): %30s vs %-30s " % ( frame.get_moment ( 5 , 50 , 'b1' , 'b1/(b2+1)' ) ,
tree .get_moment ( 5 , 50 , 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "m(5,'b1','b1/(b2+1'): %30s vs %-30s " % ( frame.moment ( 5 , 'b1' , 'b1/(b2+1)' ) ,
tree .moment ( 5 , 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "cm(5,'b1','b1/(b2+1'): %30s vs %-30s " % ( frame.central_moment ( 2 , 'b1' , 'b1/(b2+1)' ) ,
tree .central_moment ( 2 , 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "mean('b1','b1/(b2+1'): %30s vs %-30s " % ( frame.mean ( 'b1' , 'b1/(b2+1)' ) ,
tree .mean ( 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "rms ('b1','b1/(b2+1'): %30s vs %-30s " % ( frame.rms ( 'b1' , 'b1/(b2+1)' ) ,
tree .rms ( 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "skew('b1','b1/(b2+1'): %30s vs %-30s " % ( frame.skewness ( 'b1' , 'b1/(b2+1)' ) ,
tree .skewness ( 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "kurt('b1','b1/(b2+1'): %30s vs %-30s " % ( frame.kurtosis ( 'b1' , 'b1/(b2+1)' ) ,
tree .kurtosis ( 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "quantile(0.3, 'b1','b1<500'): %30s vs %-30s " % ( frame.quantile ( 0.3 , 'b1' , 'b1/(b2+1)' ) ,
tree .quantile ( 0.3 , 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "median(0.3, 'b1','b1<500'): %30s vs %-30s " % ( frame.median ( 'b1' , 'b1/(b2+1)' ) ,
tree .median ( 'b1' , 'b1/(b2+1)' ) ) )
logger.info ( "terciles('b1','b1<500'): %30s vs %-30s " % ( frame.terciles ( 'b1' , 'b1/(b2+1)' ) ,
tree .terciles ( 'b1' , 'b1/(b2+1)' ) ) )