def table ( self , title = None , prefix = ' ' ) :
"""Format weighting information as table
"""
import ostap.logger.table as Table
if title is None : title = 'Weighter(%s)' % self.__dbase
return Table.table ( self.__table , title = title , prefix = prefix ,
alignment = 'llcc' )
def derivative_testing ( der_type , func , der_exact , logger , **kwargs ) :
IMAX = der_type.IMAX
cnt1 = {}
for i in range ( IMAX ) : cnt1 [i] = SE()
ders = [ der_type ( func , I , **kwargs ) for I in range ( IMAX ) ]
with timing ( 'derivative test' , logger = logger ) :
for i in range(10000) :
x = random.uniform ( 0 , pi )
d_true = der_exact ( x )
for I , dd in enumerate ( ders ) :
delta = dd ( x ) - d_true
cnt1 [I] += delta
rows = [ ('I' , 'rms' , 'min' , 'max' ) ]
for i,c in cnt1.items() :
row = '%d' % i , '% .4g' % c.rms () , '% .4g' % c.min() , '% .4g' % c.max()
rows.append ( row )
table = T.table ( rows ,
title = 'Test numerical derivatives' ,
prefix = '# ' , alignment = 'clll' )
logger.info ( 'Test numerical derivatives\n%s' % table )
def _rd_ls_table_ ( rdir , prefix = '# ' ) :
"""Show the content of the directory as a table
>>> rdir = ...
>>> rdir.ls_table ()
"""
lines = [ ( n , k.GetClassName() ) for ( n , k ) in _rd_ikeyskeys_ ( rdir ) ]
lines.sort()
maxkey = 5
maxtype = 5
for l in lines :
maxkey = max ( maxkey , len ( l[0] ) )
maxtype = max ( maxtype , len ( l[1] ) )
fmt_type = '%%-%ds' % ( maxtype + 2 )
fmt_key = '%%-%ds' % ( maxkey + 2 )
table = [ ( fmt_key % 'Key' , fmt_type % 'type' ) ]
for line in lines :
row = fmt_key % line[0] , fmt_type % line[1]
table.append ( row )
name = rdir.GetName()
if isinstance ( rdir , ROOT.TFile ) :
name = os.path.basename ( name )
if maxkey + maxtype < len ( name ) :
name = '<.>' + name [ -maxkey - maxtype : ]
import ostap.logger.table as T
table = T.table ( table , title = '%s' % name , prefix = '# ' )
logger.info ( 'Directory %s:\n%s' % ( rdir.GetName() , table ) )
def make_print ( pdf , fitresult , title , logger = logger ) :
title2 ='%s resolution model' % title
logger.info ('%-20s : fit result\n%s' % ( title ,
fitresult.table ( title = title2 ,
prefix = '# ' ) ) )
rows = [ ( 'Parameter' , 'Value', '(Roo)Value' ) ]
row = 'mean' , '%+.6g' % pdf.get_mean () , '%+.6g' % pdf.roo_mean ()
rows.append ( row )
row = 'mode' , '%+.3g' % pdf.mode () , ''
rows.append ( row )
row = 'median' , '%+.3g' % pdf.median () , ''
rows.append ( row )
row = 'midpoint' , '%+.3g' % pdf.mid_point () , ''
rows.append ( row )
row = 'rms' , '%+.6g' % pdf.rms () , '%+.6g' % pdf.roo_rms ()
rows.append ( row )
row = 'FWHM' , '%+.6g' % pdf.fwhm () , ''
rows.append ( row )
row = 'skewness' , '%+.6g' % pdf.skewness () , '%+.6g' % pdf.roo_skewness ()
rows.append ( row )
row = 'kurtosis' , '%+.6g' % pdf.kurtosis () , '%+.6g' % pdf.roo_kurtosis ()
rows.append ( row )
table = T.table ( rows , title = title2 , prefix = '# ' )
logger.info ( 'Global features for %s\n%s' % ( title2 , table ) )
def test_derivative_5 ():
logger = getLogger ( 'test_derivative_5' )
## the function
func2 = lambda x,y : 0.25*x*x + (x-y)*(x-y) + y + 2*x
## use explicit partial derivatives
eval2_1 = Eval2VE( func2 , dFdX = lambda x,y : 2+0.5*x+2*(x-y) , dFdY = lambda x,y : 1-2*(x-y) )
## use numerical partial derivatives
eval2_2 = Eval2VE( func2 )
table = [ ( 'x' , 'y' , 'corr' , 'F(exact)' , 'F(numerical)' ) ]
for x , y in [ (0,0) , (1,0) , (0,1) , (1,2) , ( 1 , -1 ) , ( -1 , 1 ) , (2,1) ] :
for c in ( -1.0 , -0.5 , 0.0 , 0.5 , 1.0 ) :
x = VE ( x , 0.1 ** 2 )
y = VE ( y , 0.1 ** 2 )
row = [ x.toString ( '%+5.2f +/- %-5.2f' ) ,
y.toString ( '%+5.2f +/- %-5.2f' ) ,
'%+3.1f' % c ]
v1 = eval2_1 ( x , y , c )
v2 = eval2_2 ( x , y , c )
row.append ( v1.toString ( '%+6.3f +/- %-5.3f') )
row.append ( v2.toString ( '%+6.3f +/- %-5.3f') )
table.append ( tuple ( row ) )
title = 'Error propagation for F(x,y)'
table = T.table ( table , title = title , prefix = '# ' , alignment=5*'c' )
logger.info ('%s\n%s' % ( title , table ) )
def table ( self , title = 'Jobs execution statistics' , prefix = '' ) :
text = [ (' #jobs ' , '%' , ' total time' , 'time/job' , 'job server') ]
njobs = self.njobs
keys = self.__merged.keys()
for host in sorted ( keys ) :
se = self.__merged [ host ]
nj = se.njobs
time = se.time
mean = time / nj if 1 <= nj else 0.0
if 1 <= nj :
line = ( "%6d " % nj ,
" %5.1f " % ( 100. * nj / njobs ) ,
" %10.4g " % time ,
" %10.4g " % mean ,
" %-s" % host )
else :
line = "%6d "% nj , '', '' , '' , " %-s" % host
text.append ( line )
import ostap.logger.table as T
return T.table ( text , title = title , prefix = prefix )
def dump_models () :
header = 'Model' , \
'mean' , 'mode' , 'midpoint' , 'median' , \
'rms' , 'fwhm' , \
'skewness' , 'kurtosis'
mods = {}
for m in models :
mods[ m.name ] = m
rows = [ header ]
for m in sorted ( mods ) :
model = mods [ m ]
row = m , \
'%+.3g' % model.get_mean () , \
'%+.3g' % model.mode () , \
'%+.3g' % model.median () , \
'%+.3g' % model.mid_point () , \
'%+.5g' % model.rms () , \
'%+.5g' % model.fwhm () , \
'%+.5g' % model.skewness () , \
'%+.5g' % model.kurtosis ()
rows.append ( row )
table = T.table ( rows , title = "Model's features" , prefix = '# ' )
logger.info ( 'Features of models\n%s' % table )
def differences_testing ( RULE , logger ) :
a = 1.5
fun = lambda x : math.sin ( a * x )
exact_ders = ( fun , ## 0
lambda x : math.cos ( a * x ) * a , ## 1
lambda x : - math.sin ( a * x ) * a * a , ## 2
lambda x : - math.cos ( a * x ) * pow ( a , 3 ) , ## 3
lambda x : math.sin ( a * x ) * pow ( a , 4 ) , ## 4
lambda x : math.cos ( a * x ) * pow ( a , 5 ) , ## 4
lambda x : - math.sin ( a * x ) * pow ( a , 6 ) ) ## 4
h0 = 0
hmax = 1.2
N = 1000
cd = SE ()
cp = SE ()
ch = SE ()
for D in range ( 1 , RULE.DMAX + 1 ) :
table = [ ( 'I' , '#' , 'step' , '' , 'delta' , '' , 'pull' ) ]
for I in range ( RULE.IMAX ( D ) ) :
rule = RULE ( D , I , with_error = True )
for ik in range ( N ) :
z = random.uniform ( -math.pi , math.pi )
h , _ = rule.optimal_step ( fun , z , h0 , hmax )
r = rule ( fun , z , h )
exact = exact_ders [ D ] ( z )
dd = float ( r ) - exact
cd += abs ( dd )
cp += dd / r.error()
ch += h
hh = ch.mean()
dd = cd.mean()
hh , nh = pretty_ve ( hh , width = 4 , precision = 3 , parentheses = False )
dd , nd = pretty_ve ( dd , width = 4 , precision = 3 , parentheses = False )
row = '%2d' % I , \
'%2d' % len( rule.stencil) , \
hh , '' if not nh else '[10^%-3d]' % nh , \
dd , '' if not nd else '[10^%-3d]' % nd , \
"%-.2f" % cp.rms()
table.append ( row )
title = 'Finite differences test for D=%s derivative' % D
table = T.table ( table , title = title , prefix = '# ' , alignment = 'rrlclcr' )
logger.info ('%s\n%s' % ( title , table ) )
def print_stats(stats, ntoys='???'):
"""print statistics of pseudoexperiments
"""
table = [('Parameter', '#', 'mean', 'rms', '%13s / %-13s' % ('min', 'max'))
]
keys = stats.keys()
keys = sorted(keys)
def make_row(c):
n = "{:^11}".format(c.nEntries())
mean = c.mean()
mean = "%+13.6g +- %-13.6g" % (mean.value(), mean.error())
rms = "%13.6g" % c.rms()
minmax = "%+13.6g / %-+13.6g" % (c.min(), c.max())
return p, n, mean, rms, minmax
for p in sorted(stats):
if p.startswith('pull:'): continue
c = stats[p]
table.append(make_row(c))
for p in sorted(stats):
if not p.startswith('pull:'): continue
c = stats[p]
table.append(make_row(c))
import ostap.logger.table as Table
table = Table.table(table,
title="Results of %s toys" % ntoys,
alignment='lcccc',
prefix="# ")
logger.info('Results of %s toys:\n%s' % (ntoys, table))
def test_pickle():
logger = getLogger('test_pickle')
logger.info('Check pickling/unpickling')
bad = False
import pickle
rows = [('#', 'before', 'after', 'mean', 'rms')]
for i, f in enumerate(progress_bar(functions), start=1):
n, ff = f
fs = pickle.loads(pickle.dumps(ff))
s = SE()
for j in range(1000):
x = random.uniform(ff.xmin(), ff.xmax())
s += abs(fs(x) - ff(x))
mean = '%-+.6g' % s.mean()
rms = '%-+.6g' % s.rms()
if 1.e-7 < s.mean(): mean = attention(mean)
if 1.e-7 < s.rms(): rms = attention(rms)
row = '%d' % i, ff.__class__.__name__, fs.__class__.__name__, mean, rms
## row = '%d' % i , '%s' % ff , '%s' % fs , '%-+.5g' % s.mean() , '%-+.5g' % s.rms()
rows.append(row)
import ostap.logger.table as T
title = "Compare before/after serialisation"
table = T.table(rows, title=title, prefix='# ', alignment='rllll')
if bad: logger.warning('%s\n%s' % (title, table))
else: logger.info('%s\n%s' % (title, table))
def _rp_table_ ( plot , prefix = '' , title = '' ) :
"""Format <code>RooPlot</code> as a table
>>> frame = ...
>>> print ( frame.table( title = 'Title', prefix = '# ' )
- see `RooPlot`
"""
def _name ( obj ) :
n = type( obj ).__name__
p = n.find ( 'cppyy.gbl.' )
return n [ p + 10 : ] if 0 < p else n
if not title :
title = 'RooPlot %s' % plot.name
table = [ ( 'Index' , 'Type' , 'Option' , 'Name' ) ]
for index , obj in enumerate ( plot ) :
name = plot.nameOf ( index )
row = '%2d' % index , _name ( obj ) , plot.getDrawOptions ( name ) , name
table.append ( row )
import ostap.logger.table as T
return T.table ( table , title = title, prefix = prefix , alignment = 'clcw' )
def test_carlson_Eq51():
"""Test identity Eq (49) arXiv:math/9409227' )
- see Ostap::Math.carlson_RF
"""
logger = getLogger('test_carlson_Eq51')
logger.info('Test identity Eq.(451) from arXiv:math/9409227')
ad_max = -1
rd_max = -1
rows = [('Arguments', 'Left', 'Right', 'abs-delta', 'rel-delta')]
for i in range(NTEST):
x = random.uniform(0, 100)
y = random.uniform(0, 100)
p = random.uniform(0, 100)
l = random.uniform(0, 100)
m = x * y / l
a = p * p * (l + m + x + y)
b = p * (p + l) * (p + m)
left = RJ(x + l, y + l, l, p + l) + RJ(x + m, y + m, m, p + m)
right = RJ(x, y, 0, p) - 3 * RC(a, b)
ad = abs(left - right)
rd = abs(left / right - 1)
if prec_TIGHT < ad or prec_TIGHT < rd:
row = str ( (x,y,p,l,m) ) , \
'%+.12f' % left , \
'%+.12f' % right , \
'%.5g' % ad , \
'%.5g' % rd
rows.append(row)
ad_max = max(ad_max, ad)
rd_max = max(rd_max, rd)
row = '', '', '', '%.5g' % ad_max, '%.5g' % rd_max
rows.append(row)
title = 'Test Eq.(51) from arXiv:math/9409227; bad values from %d tests ' % NTEST
table = T.table(rows, title=title, prefix='# ', alignment='llll')
logger.info('%s\n%s' % (title, table))
if max(ad_max, rd_max) < prec_TIGHT:
logger.info('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
elif max(ad_max, rd_max) < prec_LOOSE:
logger.warning('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
else:
logger.error('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
def test_derivative_4 ():
logger = getLogger ( 'test_derivative_4' )
functions = (
( lambda x : cos(10.*x) , lambda x : -10*sin(10.*x) ) ,
( lambda x : x**3 , lambda x : 3.0*x*x ) ,
( lambda x : exp(x) , lambda x : exp(x) ) ,
( lambda x : x**8 , lambda x : 8.0*x**7 ) ,
( lambda x : tanh(2.*x) , lambda x : 2*(1.-tanh(2.*x)**2) ) ,
( lambda x : 1.11*x , lambda x : 1.11 ) ,
( lambda x : 1.11111 , lambda x : 0.0 ) ,
( lambda x : x**10 , lambda x : 10.*x**9 ) ,
)
from ostap.core.core import SE
counters = {}
from ostap.utils.progress_bar import progress_bar
IMAX = 8
table = [ ['Function'] + [ 'I=%d' % i for i in range ( IMAX ) ] ]
for i , o in enumerate ( progress_bar ( functions ) ) :
fun = o [ 0 ] ## function
der = o [ 1 ] ## derivative
row = [ '%2d' % (i+1) ]
for I in range ( IMAX ) :
cnt1 = SE ()
cnt2 = SE ()
dd = Derivative ( fun , step = 0.001 , calc = I , with_error = True )
for j in range ( 1000 ) :
x = random.uniform ( 0.05 , 1.5 )
res = dd ( x )
dif = float ( res ) - der ( x )
cnt1 += dif
if res.cov2() > 0 : cnt2 += dif/res.error()
mmax1 = abs ( cnt1.max () *10**12 )
if 2 < cnt2.nEntries() :
mmax2 = cnt2.max()
row.append ( '%7.3f / %-5.2fs' % ( mmax1, mmax2 ) )
else :
mmax2 = 0
row.append ( '%7.3f / %-5.2fs' % ( mmax1, mmax2 ) )
table.append ( row )
table = T.table ( table , prefix = '# ' , alignment=9*'c' )
logger.info ('Numerical differentiation: Max difference [10^12]\n%s' % table )
def test_carlson_eq19217():
"""Test identity Eq.(19.21.7) https://dlmf.nist.gov/19'
- see Ostap::Math.carlson_RD
- see Ostap::Math.carlson_RF
"""
logger = getLogger('test_carlson_eq19217')
logger.info('Test identity Eq.(19.21.7) https://dlmf.nist.gov/19')
ad_max = -1
rd_max = -1
rows = [('Arguments', 'Left', 'Right', 'abs-delta', 'rel-delta')]
for i in range(NTEST):
x = random.uniform(0, 100)
y = random.uniform(0, 100)
z = random.uniform(0, 100)
left = (x - y) * RD(y, z, x) + (z - y) * RD(x, y, z)
right = 3 * RF(x, y, z) - 3 * math.sqrt(y / (x * z))
ad = abs(left - right)
rd = abs(left / right - 1)
if prec_TIGHT < ad or prec_TIGHT < rd:
row = str ( (z,) ) , \
'%+.12f' % left , \
'%+.12f' % right , \
'%.5g' % ad , \
'%.5g' % rd
rows.append(row)
ad_max = max(ad_max, ad)
rd_max = max(rd_max, rd)
row = '', '', '', '%.5g' % ad_max, '%.5g' % rd_max
rows.append(row)
title = 'Test Eq.(19.21.7) from https://dlmf.nist.gov/19; bad values from %d tests ' % NTEST
table = T.table(rows, title=title, prefix='# ', alignment='llll')
logger.info('%s\n%s' % (title, table))
if max(ad_max, rd_max) < prec_TIGHT:
logger.info('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
elif max(ad_max, rd_max) < prec_LOOSE:
logger.warning('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
else:
logger.error('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
def test_carlson_Eq54():
"""Test identity Eq.(54) arXiv:math/9409227' )
- see Ostap::Math.carlson_RD
"""
logger = getLogger('test_carlson_Eq54')
logger.info('Test identity Eq.(54) from arXiv:math/9409227')
ad_max = -1
rd_max = -1
rows = [('Arguments', 'Left', 'Right', 'abs-delta', 'rel-delta')]
for i in range(NTEST):
x = random.uniform(0, 100)
y = random.uniform(0, 100)
z = random.uniform(0, 100)
left = RD(x, y, z) + RD(y, z, x) + RD(z, x, y)
right = 3.0 / math.sqrt(x * y * z)
ad = abs(left - right)
rd = abs(left / right - 1)
if prec_TIGHT < ad or prec_TIGHT < rd:
row = str ( (x,y,z) ) , \
'%+.12f' % left , \
'%+.12f' % right , \
'%.5g' % ad , \
'%.5g' % rd
rows.append(row)
ad_max = max(ad_max, ad)
rd_max = max(rd_max, rd)
row = '', '', '', '%.5g' % ad_max, '%.5g' % rd_max
rows.append(row)
title = 'Test Eq.(54) from arXiv:math/9409227; bad values from %d tests ' % NTEST
table = T.table(rows, title=title, prefix='# ', alignment='llll')
logger.info('%s\n%s' % (title, table))
if max(ad_max, rd_max) < prec_TIGHT:
logger.info('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
elif max(ad_max, rd_max) < prec_LOOSE:
logger.warning('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
else:
logger.error('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
def make_table(func, title, prefix="# "):
rows = [('x', 'fitted eff [%]', 'true eff [%]', 'delta [%]')]
for p in points:
e1 = 100 * func(p, error=True)
e2 = 100 * eff0(p)
d = e1 - e2
row = "%4.2f" % p , \
"%s" % e1.toString ( '(%5.2f+-%4.2f)' ) ,\
"%.2f" % e2 ,\
"%s" % d .toString ( '(%5.2f+-%4.2f)' )
rows.append(row)
from ostap.logger.table import table
return table(rows, title=title, prefix=prefix)
def _rd_table_(rdir, prefix='# '):
"""Show the content of the directory as a table
>>> rdir = ...
>>> rdir.ls_table ()
"""
lines = [(n, k.GetClassName(), k.GetObjlen())
for (n, k) in _rd_ikeyskeys_(rdir)]
lines.sort()
maxkey = 5
maxtype = 5
for l in lines:
maxkey = max(maxkey, len(l[0]))
maxtype = max(maxtype, len(l[1]))
fmt_type = '%%-%ds' % (maxtype + 2)
fmt_key = '%%-%ds' % (maxkey + 2)
table = [(fmt_key % 'Key', fmt_type % 'type', 'Size')]
for line in lines:
size = line[2]
if 1024 * 1024 * 1024 <= size:
size, _ = divmod(size, 1024 * 1024 * 1024)
size = '%s GB' % size
elif 1024 * 1024 <= size:
size, _ = divmod(size, 1024 * 1024)
size = '%s MB' % size
elif 1024 <= size:
size, _ = divmod(size, 1024)
size = '%s kB' % size
else:
size = '%3d B' % size
row = fmt_key % line[0], fmt_type % line[1], size
table.append(row)
name = rdir.GetName()
if isinstance(rdir, ROOT.TFile):
name = os.path.basename(name)
if maxkey + maxtype < len(name):
name = '<.>' + name[-maxkey - maxtype:]
import ostap.logger.table as T
return T.table(table, title='%s' % name, prefix='# ', alignment='llr')
def report_print_table(report, title='', prefix='', more_rows=[]):
"""Print a frame report data
"""
from ostap.core.core import binomEff
n0 = -1
lmax = 5
table = []
for name, passed, all in report:
n0 = max(n0, all, passed)
eff1 = binomEff(passed, all) * 100
eff2 = binomEff(passed, n0) * 100
lmax = max(len(name), lmax, len('Filter '))
item = name, passed, all, eff1, eff2
table.append(item)
lmax = max(lmax + 2, len('Selection') + 2)
fmt_name = '%%-%ds ' % lmax
fmt_input = '%10d'
fmt_passed = '%-10d'
fmt_eff = '%8.3g +- %-8.3g'
fmt_cumulated = '%8.3g +- %-8.3g'
header = (('{:^%d}' % lmax).format('Filter'), ('{:>10}').format('#input '),
('{:<10}').format(' #passed'),
('{:^20}').format('efficiency [%]'),
('{:^20}').format('cumulated efficiency [%]'))
table_data = [header]
for entry in table:
n, p, a, e1, e2 = entry
table_data.append((fmt_name % n, fmt_input % a, fmt_passed % p,
fmt_eff % (e1.value(), e1.error()),
fmt_cumulated % (e2.value(), e2.error())))
for row in more_rows:
table_data.append(row)
import ostap.logger.table as T
return T.table(table_data, title, prefix)
def show_tunnels ( tunnels = None ) :
"""Show currently opened tunnels
"""
if tunnels is None : tunnels = ppServer.open_pptunnels
rows = [ ( "local port" , 'remote host:port' ) ]
for tunnel in tunnels :
if isinstance ( tunnel , ppServer ) : row = tunnel.stamp
else : row = tunnel
rows.append ( row )
import ostap.logger.table as Table
table = Table.table ( rows , title = 'Opened %d ssh/pp-tunnels' % len ( tunnels ) , prefix = '# ')
logger.info ( 'Opened %d ssh/pp-tunnels\n%s' % ( len ( tunnels ) , table ) )
def print_gsl_errors():
"""Catch GSL errors from C++ and print the summary table at exit
"""
gsl_cnt = Ostap.Utils.GslCount
if 0 == gsl_cnt.size(): return ## No GSL errors
## get the summary
table = gsl_cnt.table()
rows = []
for tline in table:
try:
n, code, msg, reason, file, line = tline
code = int(code)
n = int(n)
except:
logger.warning(
'print_gs_errors: failure to decode line: %s, skip it!' %
str(tline))
continue
row = '%4d' % n, '%3d:%s' % (code, msg), reason, file, line
rows.append(row)
if rows:
from ostap.logger.logger import getLogger
logger = getLogger('ostap.utils.gsl')
rows = [('#', 'error', 'reason', 'file', 'line')] + rows
title = 'Summary of GSL errors'
import ostap.logger.table as T
from ostap.logger.colorized import attention
logger.error(
'%s\n%s' %
(attention(title),
T.table(rows, title=title, prefix='# ', alignment='ccccl')))
## clear the errors
gsl_cnt.clear()
del gsl_cnt
def _tab_print_(t,
title='',
prefix='',
alignment='ll',
xfmt='%+.5g',
yfmt='%+-.5g'):
"""Print interpolation table as table
>>> table= ...
>>> print ( table.table() )
"""
rows = [('Abscissa', 'Value')]
for i in range(t.size()):
x = t.x(i)
y = t.y(i)
row = xfmt % x, yfmt % y
rows.append(row)
if not title: title = 'Interpolation Table'
import ostap.logger.table as T
return T.table(rows, title=title, prefix=prefix, alignment=alignment)
def test_pickle():
logger = getLogger('test_pickle')
logger.info('Check pickling/unpickling')
import pickle
rows = [('#', 'before', 'after', 'mean', 'rms')]
for i, f in enumerate(functions, start=1):
fs = pickle.loads(pickle.dumps(f))
s = SE()
for j in range(1000):
x = random.uniform(f.xmin(), f.xmax())
s += abs(fs(x) - f(x))
row = '%d' % i, f.__class__.__name__, fs.__class__.__name__, '%-+.4g' % s.mean(
), '%-+.4g' % s.rms()
rows.append(row)
import ostap.logger.table as T
title = "Compare before/after eserialisation"
table = T.table(rows, title=title, prefix='# ', alignment='rllll')
logger.info('%s\n%s' % (title, table))
def table_style(style, prefix='', title=''):
"""Dump the style as a table"""
conf = dump_style(style)
for i in range(31):
key = 'LineStyleString_%s' % i
fmt = style.GetLineStyleString(i)
if fmt: conf[key] = fmt
table = [('#', 'Parameter', 'value')]
for i, key in enumerate(sorted(conf), start=1):
value = conf[key]
row = '%3d' % i, key, '%s' % value
table.append(row)
title = title if title else 'Style %s/%s' % (style.GetName(),
style.GetTitle())
import ostap.logger.table as T
return T.table(table, title=title, prefix=prefix, alignment='ll')
def test_derivative_2():
logger = getLogger('test_derivative_2')
cnt = {}
cntE = {}
for I in range(1, 9):
cnt[I] = SE()
func = lambda x: sin(10 * x) + x
deri = lambda x: 10 * cos(10 * x) + 1
with timing():
for i in range(10000):
x = random.uniform(0, pi)
d_true = deri(x)
for I in range(1, 9):
delta = derivative(func, x, I=I) - d_true
cnt[I] += delta
rows = [('Order', '#', 'min', 'max')]
for i in cnt:
c = cnt[I]
row = '%d' % i, '%d' % c.nEntries(), '%+.5g' % c.min(
), '%+.5g' % c.max()
rows.append(row)
table = T.table(rows,
title='Test numerical derivatives',
prefix='# ',
alignment='crll')
logger.info('Test numerical derivatives\n%s' % table)
def print_bootstrap(fitresult, stats, morevars={}, logger=logger, title=''):
"""print Bootstrap statistics
"""
header = ('Parameter', 'theta', 'theta_boot', 'bias/sigma [%]',
'error [%]')
table = []
n = 0
for name in sorted(stats):
if name in fitresult:
p = fitresult[name]
theta = p * 1.0
if not isinstance(theta, VE) or theta.cov2() <= 0:
logger.warning(
'print_bootstrap: parameter "%s" is invalid in ``fitresult'
', skip %s' % (name, theta))
continue
elif name in morevars:
theta = morevars[name]
if not isinstance(theta, VE) or theta.cov2() <= 0:
logger.warning(
'print_bootstrap: parameter "%s" is invalid in ``morevars'
', skip %s' % (name, theta))
continue
else:
continue
statistics = stats[name]
n = max(n, statistics.nEntries())
theta_boot = VE(statistics.mean().value(), statistics.mu2())
bias = theta_boot.value() - theta.value()
scale = theta.error() / theta_boot.error()
row = (name, "%+13.6g +/- %-13.6g" % (theta.value(), theta.error()),
"%+13.6g +/- %-13.6g" %
(theta_boot.value(), theta_boot.error()), '%+6.2f' %
(bias / theta.error() * 100), '%+6.2f' % (scale * 100 - 100))
table.append(row)
for name in sorted(stats):
if name in fitresult: continue
if name in morevars: continue
statistics = stats[name]
theta_boot = VE(statistics.mean().value(), statistics.mu2())
row = name, '', "%+13.6g +/- %-13.6g" % (theta_boot.value(),
theta_boot.error()), '', ''
table.append(row)
table = [header] + table
title = title if title else "Bootstrapping with #%d samples" % n
import ostap.logger.table as Table
table = Table.table(table, title=title, alignment='lcccc', prefix="# ")
logger.info('%s:\n%s' % (title, table))
def print_jackknife(fitresult, stats, morevars={}, logger=logger, title=''):
"""print Jackknife statistics
"""
header = ('Parameter', 'theta', 'theta_(.)', 'theta_jack',
'bias/sigma [%]', 'error [%]')
table = []
N = 0
for name in sorted(stats):
if name in fitresult:
p = fitresult[name]
theta = p * 1.0
if not isinstance(theta, VE) or theta.cov2() <= 0:
logger.warning(
'print_jackknife: parameter "%s" is invalid in ``fitresult'
', skip %s' % (name, theta))
continue
elif name in morevars:
theta = morevars[name]
if not isinstance(theta, VE) or theta.cov2() <= 0:
logger.warning(
'print_jackknife: parameter "%s" is invalid in ``morevars'
', skip %s' % (name, theta))
continue
else:
continue
statistics = stats[name]
N = max(N, statistics.nEntries())
## jackknife estimates
jackknife, theta_jack = jackknife_statistics(statistics, theta)
bias = theta_jack.value() - theta.value()
scale = theta.error() / theta_jack.error()
row = (name, "%+13.6g +/- %-13.6g" % (theta.value(), theta.error()),
"%+13.6g +/- %-13.6g" % (jackknife.value(), jackknife.error()),
"%+13.6g +/- %-13.6g" %
(theta_jack.value(), theta_jack.error()), '%+6.2f' %
(bias / theta.error() * 100), '%+6.2f' % (scale * 100 - 100))
table.append(row)
for name in sorted(stats):
if name in fitresult: continue
if name in morevars: continue
statistics = stats[name]
jackknife = jackknife_statistics(statistics)
row = name, '', "%+13.6g +/- %-13.6g" % (jackknife.value(),
jackknife.error()), '', '', ''
table.append(row)
table = [header] + table
title = title if title else "Jackknife results (N=%d)" % N
import ostap.logger.table as Table
table = Table.table(table, title=title, alignment='lcccccc', prefix="# ")
logger.info('%s:\n%s' % (title, table))
def _cleanup_ () :
## 1. clean up the files
tmp_files = CleanUp._tmpfiles
logger.debug ( 'remove temporary files: %s' % list ( tmp_files ) )
while tmp_files :
f = tmp_files.pop()
CleanUp.remove_file ( f )
## 2. clean up the directories
tmp_dirs = CleanUp._tmpdirs
logger.debug ( 'remove temporary directories: %s' % list ( tmp_dirs ) )
while tmp_dirs :
f = tmp_dirs.pop()
CleanUp.remove_dir ( f )
## 3.remove base directories
global base_tmp_pid_dirs
for k in base_tmp_pid_dirs :
d = base_tmp_pid_dirs [ k ]
CleanUp.remove_dir ( d )
base_tmp_pid_dirs = {}
## 4. remove base tmp directory
if for_cleanup and base_tmp_dir :
CleanUp.remove_dir ( base_tmp_dir )
if CleanUp._protected :
title = 'Kept temporary files'
rows = []
for fname in CleanUp._protected :
if os.path.exists ( fname ) and os.path.isfile ( fname ) :
row = '%s' % fname
rows.append ( row )
if rows :
rows.sort()
rows = [ ('%d' % i , f ) for i, f in enumerate ( rows , start = 1 ) ]
rows = [ ( '#', title ) ] + rows
import ostap.logger.table as T
table = T.table ( rows , title = title , prefix = "# " , alignment = 'rl' )
logger.info ( '%s:\n%s' % ( title , table ) )
if CleanUp._failed :
title = 'Not removed directories/files'
rows = []
def alldirs ( path ) :
a , b = os.path.split ( path )
yield a
while a and b :
a , b = os.path.split ( a )
yield a
pdirs = set()
for pp in CleanUp._protected :
pdirs |= set ( ( p for p in alldirs ( pp ) ) )
for fname in CleanUp._failed :
if os.path.exists ( fname ) and os.path.isdir ( fname ) and not fname in pdirs :
row = '%s' % fname
rows.append ( row )
for fname in CleanUp._failed :
if fname in CleanUp._protected : continue
if os.path.exists ( fname ) and os.path.isfile ( fname ) :
row = '%s' % fname
rows.append ( row )
if rows :
rows.sort()
rows = [ ('%d' % i , f ) for i, f in enumerate ( rows , start = 1 ) ]
rows = [ ( '#', title ) ] + rows
import ostap.logger.table as T
title = 'Not removed directories/files'
table = T.table ( rows , title = title , prefix = "# " , alignment = 'rl' )
logger.warning ( '%s\n%s' % ( title , table ) )
def frame_table(self, pattern=None, cuts='', more_vars=(), prefix=''):
"""Data frame as table
>>> frame = ...
>>> table = frame_table ( frame , '.*PT.*' , cuts = ... , more_vars = [ 'x*x/y' , 'y+z'] )
>>> print ( table )
"""
frame = DataFrame(self)
def col_type(var):
if var in cols: t = frame.GetColumnType(var)
else: return ''
if 'Double_t' == t: return 'double'
elif 'Float_t' == t: return 'float'
elif 'Bool_t' == t: return 'bool'
elif 'Char_t' == t: return 'char'
elif 'Short_t' == t: return 'short'
elif 'Int_t' == t: return 'int'
elif 'Long64_t' == t: return 'long'
elif 'UChar_t' == t: return 'unsigned char'
elif 'UShort_t' == t: return 'unisgned short'
elif 'UInt_t' == t: return 'unsigned int'
elif 'ULong64_t' == t: return 'unisgned long'
return t
## get all variables/columns
cols = tuple(frame.GetColumnNames())
vcols = cols
if pattern:
import re
cpat = re.compile(pattern)
vcols = tuple([v for v in vcols if cpat.match(v)])
svars = vcols + tuple(more_vars)
stats = fr_statVars(frame, svars, cuts=cuts, lazy=False)
if len(vcols) < 10: ifmt = '%1d.'
elif len(vcols) < 100: ifmt = '%2d.'
elif len(vcols) < 1000: ifmt = '%3d.'
else: ifmt = '%d6.'
header = ('#', 'Variable', 'Type', 'mean', 'RMS', 'min', 'max')
table = [header]
for i, e in enumerate(sorted(vcols), start=1):
stat = stats[e]
n = stat.nEntries()
mnmx = stat.minmax()
mean = stat.mean()
rms = stat.rms()
row = [
ifmt % i,
e,
col_type(e),
('%+.5g' % mean.value()).strip(), ## 4
('%-.5g' % rms).strip(), ## 5
('%+.5g' % mnmx[0]).strip(), ## 6
('%-+.5g' % mnmx[1]).strip(), ## 7
]
table.append(row)
nv = len(table)
if len(more_vars) < 10: ifmt = '%1d.'
elif len(more_vars) < 100: ifmt = '%2d.'
elif len(more_vars) < 1000: ifmt = '%3d.'
else: ifmt = '%6d.'
for i, e in enumerate(sorted(more_vars), start=1):
stat = stats[e]
n = stat.nEntries()
mnmx = stat.minmax()
mean = stat.mean()
rms = stat.rms()
row = [
ifmt % i,
e,
col_type(e),
('%+.5g' % mean.value()).strip(), ## 4
('%-.5g' % rms).strip(), ## 5
('%+.5g' % mnmx[0]).strip(), ## 6
('%-+.5g' % mnmx[1]).strip(), ## 7
]
table.append(row)
import ostap.logger.table as T
title = 'DataFrame variables'
if pattern: title = '%s (pattern %s)' % (title, pattern)
t = T.table(table, title, prefix=prefix, alignment='rllrlrl')
return t
def source_env(scripts, silent=False, lmax1=65, lmax2=55):
"""``Source'' the environment file
>>> from ostap.utuls.source_env import source_env
>>> get the dict of all modified/new variables
>>> variables = source_env ( 'my_env_script.sh' )
>>> # inspect the list and modify os.environ
>>> for key in variables :
... if k in good_keys :
... os.environ[key] = variables [ key ]
"""
from ostap.core.ostap_types import string_types
if isinstance(scripts, string_types): scripts = [scripts]
if not scripts: return
before = {}
before.update(os.environ)
variables = {}
for script in scripts:
if not os.path.exists(script) or not os.path.isfile(script):
logger.error("Invalid script ``%s'' for sourcing! skip it!" %
script)
command = 'sh -c "source %s && env"' % script
args = shlex.split(command)
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
## stderr = subprocess.PIPE )
## error = False
## for line in proc.stderr :
## if line : line = line [:-1]
## logger.error ( "stderr: %s" % line )
## error = True
## assert not error,\
## "source_env: cannot source script %s using ``%s''" % ( script , command )
vars = {}
for line in proc.stdout:
if line: line = line[:-1]
key, _, value = line.partition("=")
if _ and not key in skip:
vars[key] = value
proc.communicate()
variables.update(vars)
new_keys = set(variables.keys()) - set(before.keys())
modified = set()
for key in variables:
if key in before and variables[key] != before[key]:
modified.add(key)
if not silent and (new_keys or modified):
import ostap.logger.table as Table
if new_keys:
new_keys = list(new_keys)
new_keys.sort()
rows = [("Variable", 'Value', '#')]
for key in new_keys:
value = variables[key]
path = value.split(':')
while '' in path:
path.remove('')
path = len(path)
path = str(path) if 2 <= path else ''
value = clip(value, lmax1)
row = key, value, path
rows.append(row)
table = Table.table(rows, title="New variables", prefix='# ')
logger.info('New %d variables:%s\n%s' %
(len(new_keys), new_keys, table))
if modified:
modified = list(modified)
modified.sort()
rows = [("Variable", 'New value', '#', 'Old value', '#')]
for key in modified:
value1 = variables[key]
value2 = before[key]
path1 = value1.split(':')
while '' in path1:
path1.remove('')
path1 = len(path1)
path1 = str(path1) if 2 <= path1 else ''
path2 = value2.split(':')
while '' in path2:
path2.remove('')
path2 = len(path2)
path2 = str(path2) if 2 <= path2 else ''
value1 = clip(value1, lmax2)
value2 = clip(value2, lmax2)
row = key, value1, path1, value2, path2
rows.append(row)
table = Table.table(rows, title="Modified variables", prefix='# ')
logger.info('Modified %d variables:%s\n%s' %
(len(modified), modified, table))
result = {}
for key in new_keys:
result[key] = variables[key]
for key in modified:
result[key] = variables[key]
return result
def ls(self, pattern='', load=True):
"""List the available keys (patterns included).
Pattern matching is performed accoriding to
fnmatch/glob/shell rules [it is not regex!]
>>> db = ...
>>> db.ls() ## all keys
>>> db.ls ('*MC*')
"""
n = os.path.basename(self.filename)
ap = os.path.abspath(self.filename)
try:
fs = os.path.getsize(self.filename)
except:
fs = -1
if fs < 0: size = "???"
elif fs < 1024: size = str(fs)
elif fs < 1024 * 1024:
size = '%.2fkB' % (float(fs) / 1024)
elif fs < 1024 * 1024 * 1024:
size = '%.2fMB' % (float(fs) / (1024 * 1024))
else:
size = '%.2fGB' % (float(fs) / (1024 * 1024 * 1024))
keys = []
for k in self.ikeys(pattern):
keys.append(k)
keys.sort()
if keys: mlen = max([len(k) for k in keys]) + 2
else: mlen = 2
fmt = ' --> %%-%ds : %%s' % mlen
table = [('Key', 'type', ' size ')]
for k in keys:
size = ''
ss = self.__sizes.get(k, -1)
if ss < 0: size = ''
elif ss < 1024: size = '%7d ' % ss
elif ss < 1024 * 1024:
size = '%7.2f kB' % (float(ss) / 1024)
elif ss < 1024 * 1024 * 1024:
size = '%7.2f MB' % (float(ss) / (1024 * 1024))
else:
size = '%7.2f GB' % (float(ss) / (1024 * 1024 * 1024))
ot = type(self[k])
otype = ot.__cppname__ if hasattr(ot,
'__cppname__') else ot.__name__
row = '{:15}'.format(k), '{:15}'.format(otype), size
table.append(row)
import ostap.logger.table as T
t = self.__class__.__name__
title = '%s:%s' % (t, n)
maxlen = 0
for row in table:
rowlen = 0
for i in row:
rowlen += len(i)
maxlen = max(maxlen, rowlen)
if maxlen + 3 <= len(title):
title = '<.>' + title[-maxlen:]
table = T.table(table, title=title, prefix='# ')
ll = getLogger(n)
line = 'Database %s:%s #keys: %d size: %s' % (t, ap, len(self), size)
ll.info('%s\n%s' % (line, table))
