def print_gsl_errors():
"""Catch GSL errors from C++ and print the summary table at exit
"""
gsl_cnt = Ostap.Utils.GslCount
if 0 == gsl_cnt.size(): return ## No GSL errors
## get the summary
table = gsl_cnt.table()
rows = []
for tline in table:
try:
n, code, msg, reason, file, line = tline
code = int(code)
n = int(n)
except:
logger.warning(
'print_gs_errors: failure to decode line: %s, skip it!' %
str(tline))
continue
row = '%4d' % n, '%3d:%s' % (code, msg), reason, file, line
rows.append(row)
if rows:
from ostap.logger.logger import getLogger
logger = getLogger('ostap.utils.gsl')
rows = [('#', 'error', 'reason', 'file', 'line')] + rows
title = 'Summary of GSL errors'
import ostap.logger.table as T
from ostap.logger.colorized import attention
logger.error(
'%s\n%s' %
(attention(title),
T.table(rows, title=title, prefix='# ', alignment='ccccl')))
## clear the errors
gsl_cnt.clear()
del gsl_cnt
def _rfr_table_ ( r , title = '' , prefix = '' , more_vars = {} ) :
""" print RooFitResult as a table
>>> result = ...
>>> result.table()
"""
from ostap.fitting.utils import fit_status, cov_qual
rows = []
## 1. fit status
status = r.status()
if status :
row = attention ( 'Status' ) , '' , attention ( fit_status ( status ) ) , ''
rows.append ( row )
else :
row = 'Status' , '' , allright ( fit_status ( status ) ) , ''
rows.append ( row )
## 2. minumum NLL
s , n = pretty_float ( r.minNll() )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Minimized FCN/NLL value" , n , ' ' + s , '' ) )
s , n = pretty_float ( r.edm () )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( 'Estimated distance to minimum' , n , ' ' + s , '' ) )
cq = r.covQual()
cn = ''
if -1 == cq :
cn = cov_qual ( cq )
elif 3 == cq :
cn = allright ( cov_qual ( cq ) )
elif cq in ( 0 , 1 , 2 ) :
cn = attention ( cov_qual ( cq ) )
else :
cn = cov_qual ( cq )
rows.append ( ( 'Covariance matrix quality' , '' , ' ' + cn , '' ) )
for i in range ( r.numStatusHistory() ) :
label = r.statusLabelHistory ( i )
code = r.statusCodeHistory ( i )
row = 'Status: %s '% label , '' , '%d' % code
if not code in ( 0 , -1 ) :
row = attention ( row [ 0 ] ) , row [ 1 ] , ' ' + attention ( row [ 2 ] ) , ''
else :
row = row [ 0 ] , row [ 1 ] , ' ' + allright ( row [ 2 ] ) , ''
rows.append ( row )
nbadnll = r.numInvalidNLL()
if 0 < nbadnll :
rows.append ( ( 'Invalid FCN/NLL evaluations' , '' , ' %d' % nbadnll , '' ) )
with_globcorr = not ( (6,24) <= root_info < (6,28) )
if with_globcorr : rows = [ ( '', 'Unit', 'Value' , 'Global/max correlation [%]') ] + rows
else : rows = [ ( '', 'Unit', 'Value' , 'Max correlation [%]') ] + rows
pars_all = r.params ( float_only = False )
pars_float = r.params ( float_only = True )
## constant/fix parameters
crows = []
for p in pars_all :
if p in pars_float : continue
v , a = pars_all [ p ]
s , n = pretty_float ( v.value() )
if n : n = '[10^%+d]' % n
else : n = ''
row = p , n , ' ' + s + ' (fix)' , ''
crows.append ( row )
## floating parameters
max_corr = False
frows = []
for p in pars_float :
v , a = pars_float [ p ]
if not a.hasAsymError() :
s , n = pretty_ve ( v )
else :
s , n = pretty_2ve ( a.getVal() , a.getAsymErrorHi() , a.getAsymErrorLo() )
if n : n = '[10^%+d]' % n
else : n = ''
if 0 <= cq and 1 < len ( pars_float ) :
mxr , mxv = r.max_cor ( p )
if with_globcorr :
gc = -1.0
gc = r.globalCorr ( p ) if 3 == cq else -1.00
if 0 <= gc : cc = '% +5.1f/(% +5.1f,%s)' % ( gc*100 , mxr*100 , mxv )
else : cc = '% +5.1f : %-s' % ( mxr * 100 , mxv )
if 0.95 < abs ( mxr ) or 0.95 < gc : cc = attention ( cc )
else :
cc = '% +5.1f : %-s' % ( mxr * 100 , mxv )
if 0.95 < abs ( mxr ) : cc = attention ( cc )
max_corr = True
row = p , n , s , cc
else :
row = p , n , s
frows.append ( row )
## more parameters
mrows = []
for p in sorted ( more_vars ) :
func = more_vars [ p ]
v = func ( r )
s , n = pretty_ve ( v )
if n : n = '[10^%+d]' % n
else : n = ''
cc = 'derived'
row = p , n , s , cc
mrows.append ( row )
crows.sort()
frows.sort()
all = rows + crows + frows + mrows
import ostap.logger.table as T
return T.table ( all , title = title if title else r.GetTitle() , prefix = prefix , alignment = 'llll' )
def makeWeights(
dataset,
plots=[],
database="weights.db",
compare=None, ## comparison function
delta=0.01, ## delta for ``mean'' weight variation
minmax=0.03, ## delta for ``minmax'' weight variation
power=None, ## auto-determination
debug=True, ## save intermediate information in DB
make_plots=False, ## make plots
tag="Reweighting"):
"""The main function: perform one re-weighting iteration
and reweight ``MC''-data set to looks as ``data''(reference) dataset
>>> results = makeWeights (
... dataset , ## data source to be reweighted (DataSet, TTree, abstract source)
... plots , ## reweighting plots
... database , ## datadabse to store/update reweigting results
... delta , ## stopping criteria for `mean` weight variation
... minmax , ## stopping criteria for `min/max` weight variation
... power , ## effective power to apply to the weigths
... debug = True , ## store debuig information in database
... make_plots = True , ## produce useful comparison plots
... tag = 'RW' ) ## tag for better printout
If `make_plots = False`, it returns the tuple of active reweitings:
>>> active = makeWeights ( ... , make_plots = False , ... )
Otherwise it also returns list of comparison plots
>>> active, cmp_plots = makeWeights ( ... , make_plots = True , ... )
>>> for item in cmp_plots :
... what = item.what
... hdata = item.data
... hmc = item.mc
... hweight = item.weight
If no more rewighting iteratios required, <code>active</code> is an empty tuple
"""
assert 0 < delta, "makeWeights(%s): Invalid value for ``delta'' %s" % (
tag, delta)
assert 0 < minmax, "makeWeights(%s): Invalid value for ``minmax'' %s" % (
tag, minmax)
from ostap.logger.colorized import allright, attention, infostr
from ostap.utils.basic import isatty
nplots = len(plots)
## if 1 < nplots :
## import math
## fudge_factor = math.sqrt ( 1.0 / max ( 2.0 , nplots - 1.0 ) )
## delta = delta * fudge_factor
## minmax = minmax * fudge_factor
## list of plots to compare
cmp_plots = []
## reweighting summary table
header = ('Reweighting', 'wmin/wmax', 'OK?', 'wrms[%]', 'OK?', 'chi2/ndf',
'ww', 'exp')
rows = {}
save_to_db = []
## number of active plots for reweighting
for wplot in plots:
what = wplot.what ## variable/function to plot/compare
how = wplot.how ## weight and/or additional cuts
address = wplot.address ## address in database
hdata0 = wplot.data ## original "DATA" object
hmc0 = wplot.mc_histo ## original "MC" histogram
ww = wplot.w ## relative weight
projector = wplot.projector ## projector for MC data
ignore = wplot.ignore ## ignore for weigtht building?
#
# normalize the data
#
hdata = hdata0
if isinstance(hdata, ROOT.TH1): hdata = hdata.density()
# =====================================================================
## make a plot on (MC) data with the weight
# =====================================================================
hmc0 = projector(dataset, hmc0, what, how)
st = hmc0.stat()
mnmx = st.minmax()
if iszero(mnmx[0]):
logger.warning("%s: statistic goes to zero %s/``%s''" %
(tag, st, address))
elif mnmx[0] <= 0:
logger.warning("%s: statistic is negative %s/``%s''" %
(tag, st, address))
# =====================================================================
## normalize MC
# =====================================================================
hmc = hmc0.density()
# =====================================================================
## calculate the reweighting factor : a bit conservative (?)
# this is the only important line
# =====================================================================
# try to exploit finer binning if/when possible
hboth = isinstance(hmc, ROOT.TH1) and isinstance(hdata, ROOT.TH1)
if hboth and 1 == hmc.dim () and 1 == hdata.dim () and \
len ( hmc ) >= len( hdata ) :
w = (1.0 / hmc) * hdata ## NB!
elif hboth and 2 == hmc.dim () and 2 == hdata.dim () and \
( hmc.binsx() >= hdata.binsx() ) and \
( hmc.binsy() >= hdata.binsy() ) :
w = (1.0 / hmc) * hdata ## NB!
elif hboth and 3 == hmc.dim () and 3 == hdata.dim () and \
( hmc.binsx() >= hdata.binsx() ) and \
( hmc.binsy() >= hdata.binsy() ) and \
( hmc.binsz() >= hdata.binsz() ) :
w = (1.0 / hmc) * hdata ## NB!
else:
w = hdata / hmc ## NB!
# =====================================================================
## scale & get the statistics of weights
w /= w.stat().mean().value()
cnt = w.stat()
#
mnw, mxw = cnt.minmax()
wvar = cnt.rms() / cnt.mean()
good1 = wvar.value() <= delta
good2 = abs(mxw - mnw) <= minmax
good = good1 and good2 ## small variance?
#
c2ndf = 0
for i in w:
c2ndf += w[i].chi2(1.0)
c2ndf /= (len(w) - 1)
## build the row in the summary table
row = address , \
'%-5.3f/%5.3f' % ( cnt.minmax()[0] , cnt.minmax()[1] ) , \
allright ( '+' ) if good2 else attention ( '-' ) , \
(wvar * 100).toString('%6.2f+-%-6.2f') , \
allright ( '+' ) if good1 else attention ( '-' ) , '%6.2f' % c2ndf
## make plots at the start of each iteration?
if make_plots:
item = ComparisonPlot(what, hdata, hmc, w)
cmp_plots.append(item)
row = tuple(list(row) + ['%4.3f' % ww if 1 != ww else ''])
rows[address] = row
#
## make decision based on the variance of weights
#
mnw, mxw = cnt.minmax()
if (not good) and (not ignore): ## small variance?
save_to_db.append((address, ww, hdata0, hmc0, hdata, hmc, w))
# =====================================================================
## make a comparison (if needed)
# =====================================================================
if compare: compare(hdata0, hmc0, address)
active = tuple([p[0] for p in save_to_db])
nactive = len(active)
if power and callable(power):
eff_exp = power(nactive)
elif isinstance(power, num_types) and 0 < power <= 1.5:
eff_exp = 1.0 * power
elif 1 == nactive and 1 < len(plots):
eff_exp = 0.95
elif 1 == nactive:
eff_exp = 1.00
else:
eff_exp = 1.10 / max(nactive, 1)
while database and save_to_db:
entry = save_to_db.pop()
address, ww, hd0, hm0, hd, hm, weight = entry
cnt = weight.stat()
mnw, mxw = cnt.minmax()
## avoid too large or too small weights
for i in weight:
w = weight[i]
if w.value() < 0.5:
weight[i] = VE(0.5, w.cov2())
elif w.value() > 2.0:
weight[i] = VE(2.0, w.cov2())
if 1 < nactive and 1 != ww:
eff_exp *= ww
logger.info("%s: apply ``effective exponent'' of %.3f for ``%s''" %
(tag, eff_exp, address))
if 1 != eff_exp and 0 < eff_exp:
weight = weight**eff_exp
row = list(rows[address])
row.append('%4.3f' % eff_exp)
rows[address] = tuple(row)
with DBASE.open(database) as db:
db[address] = db.get(address, []) + [weight]
if debug:
addr = address + ':REWEIGHTING'
db[addr] = db.get(addr, []) + list(entry[2:])
del hd0, hm0, hd, hm, weight, entry
table = [header]
for row in rows:
table.append(rows[row])
import ostap.logger.table as Table
logger.info(
'%s, active:#%d \n%s ' %
(tag, nactive,
Table.table(table, title=tag, prefix='# ', alignment='lccccccc')))
cmp_plots = tuple(cmp_plots)
return (active, cmp_plots) if make_plots else active
from ostap.logger.colorized import allright, attention
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ : logger = getLogger( 'ostap.fitting.minuit' )
else : logger = getLogger( __name__ )
# =============================================================================
logger.debug ( 'Some useful decorations for (T)Minuit functions')
# =============================================================================
partypes = integer_types
# =============================================================================
## return codes from MINUIT commands
return_codes = {
0 : allright ( 'command executed normally' ) ,
1 : attention ( 'command is blank, ignored' ) ,
2 : attention ( 'command line unreadable, ignored' ) ,
3 : attention ( 'unknown command, ignored' ) ,
4 : attention ( 'abnormal termination (e.g., MIGRAD not converged)' ),
5 : 'command is a request to read PARAMETER definitions' ,
6 : "'SET INPUT' command" ,
7 : "'SET TITLE' command" ,
8 : "'SET COVAR' command" ,
9 : 'reserved' ,
10 : 'END command' ,
11 : 'EXIT or STOP command' ,
12 : 'RETURN command' ,
}
# =============================================================================
## get the parameter from (T)Minuit
# @code
def makeWeights(
dataset,
plots=[],
database="weights.db",
compare=None, ## comparison function
delta=0.001, ## delta for ``mean'' weight variation
minmax=0.05, ## delta for ``minmax'' weight variation
power=0, ## auto-determination
debug=True, ## save intermediate information in DB
tag="Reweighting"):
assert 0 < delta, "makeWeights(%s): Invalid value for ``delta'' %s" % (
tag, delta)
assert 0 < minmax, "makeWeights(%s): Invalid value for ``minmax'' %s" % (
tag, minmax)
from ostap.logger.colorized import allright, attention, infostr
from ostap.utils.basic import isatty
power = power if power >= 1 else len(plots)
nplots = len(plots)
if 1 < nplots:
import math
fudge_factor = math.sqrt(1.0 / max(2.0, nplots - 1.0))
delta = delta * fudge_factor
minmax = minmax * fudge_factor
save_to_db = []
## number of active plots for reweighting
for wplot in plots:
what = wplot.what ## variable/function to plot/compare
how = wplot.how ## weight and/or additional cuts
address = wplot.address ## address in database
hdata0 = wplot.data ## original "DATA" object
hmc0 = wplot.mc_histo ## original "MC" histogram
ww = wplot.w ## relative weight
#
# normailze the data
#
hdata = hdata0
if isinstance(hdata, ROOT.TH1): hdata = hdata.density()
# =====================================================================
## make a plot on (MC) data with the weight
# =====================================================================
dataset.project(hmc0, what, how)
st = hmc0.stat()
mnmx = st.minmax()
if iszero(mnmx[0]):
logger.warning("Reweighting: statistic goes to zero %s/``%s''" %
(st, address))
# =====================================================================
## normalize MC
# =====================================================================
hmc = hmc0.density()
# =====================================================================
## calculate the reweighting factor : a bit conservative (?)
# this is the only important line
# =====================================================================
# try to exploit finer binning if/when possible
if isinstance ( hmc , ( ROOT.TH1F , ROOT.TH1D ) ) and \
isinstance ( hdata , ( ROOT.TH1F , ROOT.TH1D ) ) and \
len ( hmc ) >= len( hdata ) :
w = (1.0 / hmc) * hdata ## NB!
## elif isinstance ( hmc , ( ROOT.TH2F , ROOT.TH2D ) ) and \
## isinstance ( hdata , ( ROOT.TH2F , ROOT.TH2D ) ) and \
## len ( hmc.GetXaxis() ) >= len( hdata.GetXaxis () ) and \
## len ( hmc.GetYaxis() ) >= len( hdata.GetYaxis () ) : w = ( 1.0 / hmc ) * hdata ## NB!
## elif isinstance ( hmc , ( ROOT.TH3F , ROOT.TH3D ) ) and \
## isinstance ( hdata , ( ROOT.TH3F , ROOT.TH3D ) ) and \
## len ( hmc.GetXaxis() ) >= len( hdata.GetXaxis () ) and \
## len ( hmc.GetYaxis() ) >= len( hdata.GetYaxis () ) and \
## len ( hmc.GetZaxis() ) >= len( hdata.GetZaxis () ) : w = ( 1.0 / hmc ) * hdata ## NB!
else:
w = hdata / hmc ## NB!
# =====================================================================
## scale & get the statistics of weights
w /= w.stat().mean().value()
cnt = w.stat()
#
mnw, mxw = cnt.minmax()
wvar = cnt.rms() / cnt.mean()
good1 = wvar.value() <= delta
good2 = abs(mxw - mnw) <= minmax
good = good1 and good2 ## small variance?
#
afunc1 = allright if good1 else attention
afunc2 = allright if good2 else attention
#
message = "%s: %24s:" % (tag, address)
message += ' ' + 'mean=%12s' % cnt.mean().toString('(%4.2f+-%4.2f)')
message += ' ' + afunc2('min/max=%-5.3f/%5.3f' %
(cnt.minmax()[0], cnt.minmax()[1]))
message += ' ' + afunc1('rms=%s[%%]' %
(wvar * 100).toString('(%4.2f+-%4.2f)'))
logger.info(message)
#
## make decision based on the variance of weights
#
mnw, mxw = cnt.minmax()
if good: ## small variance?
message = "%s: No more reweights for %s" % (tag, address)
message += ' ' + allright("min/max/rms=%+3.1f/%+3.1f/%3.1f[%%]" %
((mnw - 1) * 100,
(mxw - 1) * 100, 100 * wvar))
logger.info(message)
del w, hdata, hmc
else:
save_to_db.append((address, ww, hdata0, hmc0, hdata, hmc, w))
# =====================================================================
## make a comparison (if needed)
# =====================================================================
if compare: compare(hdata0, hmc0, address)
## for single reweighting
## if 1 == nplots : power = 1
## if power != nplots :
# logger.info ( "%s: ``power'' is %g/#%d" % ( tag , power , nplots ) )
active = [p[0] for p in save_to_db]
all = [p.address for p in plots]
for i, a in enumerate(all):
if a in active:
if isatty(): all[i] = attention(a)
else: all[i] = '*' + a + '*'
else:
if isatty(): all[i] = allright(a)
logger.info("%s: reweights are: %s" % (tag, (', '.join(all))))
## if len ( active ) != nplots :
## if database and save_to_db :
## power += ( nplots - len ( active ) )
## logger.info ("%s: ``power'' is changed to %g" % ( tag , power ) )
nactive = len(active)
while database and save_to_db:
entry = save_to_db.pop()
address, ww, hd0, hm0, hd, hm, weight = entry
## eff_exp = 1.0 / power
## eff_exp = 0.95 / ( 1.0 * nactive ) ** 0.5
cnt = weight.stat()
mnw, mxw = cnt.minmax()
if 0.95 < mnw and mxw < 1.05:
eff_exp = 0.75 if 1 < nactive else 1.50
elif 0.90 < mnw and mxw < 1.10:
eff_exp = 0.70 if 1 < nactive else 1.30
elif 0.80 < mnw and mxw < 1.20:
eff_exp = 0.65 if 1 < nactive else 1.25
elif 0.70 < mnw and mxw < 1.30:
eff_exp = 0.60 if 1 < nactive else 1.15
elif 0.50 < mnw and mxw < 1.50:
eff_exp = 0.55 if 1 < nactive else 1.10
else:
eff_exp = 0.50 if 1 < nactive else 1.0
## print 'effective exponent is:', eff_exp , address , mnw , mxw , (1.0/mnw)*mnw**eff_exp , (1.0/mxw)*mxw**eff_exp
if 1 < nactive and 1 != ww:
eff_exp *= ww
logger.info("%s: apply ``effective exponent'' of %.3f for ``%s''" %
(tag, eff_exp, address))
if 1 != eff_exp and 0 < eff_exp:
weight = weight**eff_exp
## print 'WEIGHT stat', eff_exp, weight.stat()
## hmmmm... needed ? yes!
#if 1 < power : weight = weight ** ( 1.0 / power )
## relative importance
#if 1 != ww :
# logger.info ("%s: apply ``relative importance factor'' of %.3g for ``'%s'" % ( tag , ww , address ) )
# weight = weight ** ww
with DBASE.open(database) as db:
db[address] = db.get(address, []) + [weight]
if debug:
addr = address + ':REWEIGHTING'
db[addr] = db.get(addr, []) + list(entry[2:])
del hd0, hm0, hd, hm, weight, entry
return active
def test_carlson_values():
"""Test predefined values, (section 3), arXiv:math/9409227
"""
logger = getLogger('test_carlson_values')
logger.info('Test predefined values, (section 3), arXiv:math/9409227')
test_RF = [
('RF', RF, (1, 2, 0), 1.3110287771461),
('RF', RF, (2, 3, 4), 0.58408284167715),
## ( 'RF' , RF , (1,2,4) , 0.6850858166 ) , ## extra
('RF', RF, (1, 2, 4), 0.6850858166334359), ## extra
##
('RF_int', RF_int, (1, 2, 0), 1.3110287771461),
('RF_int', RF_int, (2, 3, 4), 0.58408284167715),
## ( 'RF_int' , RF_int , (1,2,4) , 0.6850858166 ) , ## extra
('RF_int', RF_int, (1, 2, 4), 0.6850858166334359), ## extra
##
('RF_gsl', RF_gsl, (1, 2, 0), 1.3110287771461),
('RF_gsl', RF_gsl, (2, 3, 4), 0.58408284167715),
## ( 'RF_gsl' , RF_gsl , (1,2,4) , 0.6850858166 ) , ## extra
('RF_gsl', RF_gsl, (1, 2, 4), 0.6850858166334359), ## extra
##
('RF2', RF, (1, 2), 1.3110287771461), ## 2-argument form
]
test_RC = [
('RC', RC, (0, 0.25), math.pi),
('RC', RC, (0.25 * 9, 2), math.log(2.0)),
('RC', RC, (0.25, -2), math.log(2.0) / 3.0),
##
('RC_gsl', RC_gsl, (0, 0.25), math.pi),
('RC_gsl', RC_gsl, (0.25 * 9, 2), math.log(2.0)),
('RC_gsl', RC_gsl, (0.25, -2), math.log(2.0) / 3.0),
##
('RC_int', RC_int, (0, 0.25), math.pi),
('RC_int', RC_int, (0.25 * 9, 2), math.log(2.0)),
('RC_int', RC_int, (0.25, -2), math.log(2.0) / 3.0),
]
test_RJ = [
('RJ', RJ, (0, 1, 2, 3), 0.77688623778582),
('RJ', RJ, (2, 3, 4, 5), 0.14297579667157),
##
('RJ_gsl', RJ_gsl, (0, 1, 2, 3), 0.77688623778582),
('RJ_gsl', RJ_gsl, (2, 3, 4, 5), 0.14297579667157),
##
('RJ_int', RJ_int, (0, 1, 2, 3), 0.77688623778582),
('RJ_int', RJ_int, (2, 3, 4, 5), 0.14297579667157),
]
test_RD = [
('RD', RD, (0, 2, 1), 1.7972103521034),
('RD', RD, (2, 3, 4), 0.16510527294261),
##
('RD_gsl', RD_gsl, (0, 2, 1), 1.7972103521034),
('RD_gsl', RD_gsl, (2, 3, 4), 0.16510527294261),
##
('RD_int', RD_int, (0, 2, 1), 1.7972103521034),
('RD_int', RD_int, (2, 3, 4), 0.16510527294261),
]
test_RG = [
('RG', RG, (0, 16, 16), math.pi),
('RG', RG, (2, 3, 4), 1.7255030280692),
('RG', RG, (0, 0.0796, 4), 1.0284758090288),
##
('RG_gsl', RG_gsl, (0, 16, 16), math.pi),
('RG_gsl', RG_gsl, (2, 3, 4), 1.7255030280692),
('RG_gsl', RG_gsl, (0, 0.0796, 4), 1.0284758090288),
##
('RG_int', RG_int, (0, 16, 16), math.pi),
('RG_int', RG_int, (2, 3, 4), 1.7255030280692),
('RG_int', RG_int, (0, 0.0796, 4), 1.0284758090288),
##
('RG2', RG, (16, 16), math.pi), ## 2-argument form
('RG2', RG, (0.0796, 4), 1.0284758090288), ## 2-argument form
]
rows = [('Function', 'Arguments', 'Result', 'Expected', 'abs-delta',
'rel-delta')]
ad_max = -1
rd_max = -1
with gslCount():
for test in test_RF + test_RC + test_RJ + test_RD + test_RG:
name, fun, args, r = test
result = fun(*args)
ad = abs(result - r)
rd = abs(result / r - 1)
at, rt = '%.4g' % ad, '%.4g' % rd
if prec_TIGHT < ad: at = attention(at)
if prec_TIGHT < rd: rt = attention(rt)
row = name, str(args), '%+.12f' % result, '%+.12f' % r, at, rt
rows.append(row)
ad_max = max(ad_max, ad)
rd_max = max(rd_max, rd)
table = T.table(rows,
title='Test of Carlson forms',
prefix='# ',
alignment='llllll')
logger.info('Test Carlson forms:\n%s' % table)
if max(ad_max, rd_max) < prec_TIGHT:
logger.info('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
elif max(ad_max, rd_max) < prec_LOOSE:
logger.warning('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
else:
logger.error('Maximal differences are %.5g/%.5g (abs/rel)' %
(ad_max, rd_max))
def test_carlson_cmp():
"""Compare local/GSL and plain integration methods
"""
logger = getLogger('test_carlson_1')
logger.info('Compare local/GSL and plain integration methods ')
cF1 = SE()
cF2 = SE()
cD1 = SE()
cD2 = SE()
cC1 = SE()
cC2 = SE()
cJ1 = SE()
cJ2 = SE()
cG1 = SE()
cG2 = SE()
with gslCount():
for i in range(10000):
x = random.uniform(0, 1)
y = random.uniform(0, 1)
z = random.uniform(0, 1)
p = random.uniform(0, 1)
## RF
v1 = RF(x, y, z)
v2 = RF_gsl(x, y, z)
v3 = RF_int(x, y, z)
cF1 += (v1 / v2) - 1
cF2 += (v1 / v3) - 1
v1 = RF(x, y, p)
v2 = RF_gsl(x, y, p)
v3 = RF_int(x, y, p)
cF1 += (v1 / v2) - 1
cF2 += (v1 / v3) - 1
v1 = RF(x, p, z)
v2 = RF_gsl(x, p, z)
v3 = RF_int(x, p, z)
cF1 += (v1 / v2) - 1
cF2 += (v1 / v3) - 1
v1 = RF(p, x, z)
v2 = RF_gsl(p, x, z)
v3 = RF_int(p, x, z)
cF1 += (v1 / v2) - 1
cF2 += (v1 / v3) - 1
## RJ
v1 = RJ(x, y, z, p)
v2 = RJ_gsl(x, y, z, p)
v3 = RJ_int(x, y, z, p)
cJ1 += (v1 / v2) - 1
cJ2 += (v1 / v3) - 1
v1 = RJ(x, y, p, z)
v2 = RJ_gsl(x, y, p, z)
v3 = RJ_int(x, y, p, z)
cJ1 += (v1 / v2) - 1
cJ2 += (v1 / v3) - 1
v1 = RJ(x, p, z, y)
v2 = RJ_gsl(x, p, z, y)
v3 = RJ_int(x, p, z, y)
cJ1 += (v1 / v2) - 1
cJ2 += (v1 / v3) - 1
v1 = RJ(p, y, z, x)
v2 = RJ_gsl(p, y, z, x)
v3 = RJ_int(p, y, z, x)
cJ1 += (v1 / v2) - 1
cJ2 += (v1 / v3) - 1
## RD
v1 = RD(x, y, z)
v2 = RD_gsl(x, y, z)
v3 = RD_int(x, y, z)
cD1 += (v1 / v2) - 1
cD2 += (v1 / v3) - 1
v1 = RD(x, y, p)
v2 = RD_gsl(x, y, p)
v3 = RD_int(x, y, p)
cD1 += (v1 / v2) - 1
cD2 += (v1 / v3) - 1
v1 = RD(x, p, z)
v2 = RD_gsl(x, p, z)
v3 = RD_int(x, p, z)
cD1 += (v1 / v2) - 1
cD2 += (v1 / v3) - 1
v1 = RD(p, y, z)
v2 = RD_gsl(p, y, z)
v3 = RD_int(p, y, z)
cD1 += (v1 / v2) - 1
cD2 += (v1 / v3) - 1
## RC
v1 = RC(x, y)
v2 = RC_gsl(x, y)
v3 = RC_int(x, y)
cC1 += (v1 / v2) - 1
cC2 += (v1 / v2) - 1
v1 = RC(x, z)
v2 = RC_gsl(x, z)
v3 = RC_int(x, z)
cC1 += (v1 / v2) - 1
cC2 += (v1 / v2) - 1
v1 = RC(x, p)
v2 = RC_gsl(x, p)
v3 = RC_int(x, p)
cC1 += (v1 / v2) - 1
cC2 += (v1 / v2) - 1
v1 = RC(y, z)
v2 = RC_gsl(y, z)
v3 = RC_int(y, z)
cC1 += (v1 / v2) - 1
cC2 += (v1 / v2) - 1
v1 = RC(y, p)
v2 = RC_gsl(y, p)
v3 = RC_int(y, p)
cC1 += (v1 / v2) - 1
cC2 += (v1 / v2) - 1
v1 = RC(z, p)
v2 = RC_gsl(z, p)
v3 = RC_int(z, p)
cC1 += (v1 / v2) - 1
cC2 += (v1 / v2) - 1
## RG
v1 = RG(x, y, z)
v2 = RG_gsl(x, y, z)
v3 = RG_int(x, y, z)
cG1 += (v1 / v2) - 1
cG2 += (v1 / v2) - 1
v1 = RG(x, y, p)
v2 = RG_gsl(x, y, p)
v3 = RG_int(x, y, p)
cG1 += (v1 / v2) - 1
cG2 += (v1 / v2) - 1
v1 = RG(x, p, z)
v2 = RG_gsl(x, p, z)
v3 = RG_int(x, p, z)
cG1 += (v1 / v2) - 1
cG2 += (v1 / v2) - 1
v1 = RG(p, y, z)
v2 = RG_gsl(p, y, z)
v3 = RG_int(p, y, z)
cG1 += (v1 / v2) - 1
cG2 += (v1 / v2) - 1
rows = [('Name', '#', 'Mean', 'rms', 'min', 'max')]
for n, c in [('RF', cF1), ('RJ', cJ1), ('RD', cD1), ('RC', cC1),
('RG', cG1)]:
mean = c.mean()
tmean = '%+.5g' % mean
if prec_TIGHT < abs(mean): tmean = attention(tmean)
rms = c.rms()
trms = '%+.5g' % rms
if prec_TIGHT < abs(rms): trms = attention(trms)
vmin = c.min()
tmin = '%+.5g' % vmin
if prec_TIGHT < abs(vmin): tmin = attention(tmin)
vmax = c.max()
tmax = '%+.5g' % vmax
if prec_TIGHT < abs(vmax): tmax = attention(tmax)
row = n, '%d' % c.nEntries(), tmean, trms, tmin, tmax
rows.append(row)
table = T.table(rows,
title='Test of Carlson forms, Local vs GSL ',
prefix='# ',
alignment='lrllll')
logger.info('Test Carlson forms, Local vs GSL:\n%s' % table)
rows = [('Name', '#', 'Mean', 'rms', 'min', 'max')]
for n, c in [('RF', cF2), ('RJ', cJ2), ('RD', cD2), ('RC', cC2),
('RG', cG2)]:
mean = c.mean()
tmean = '%+.5g' % mean
if prec_TIGHT < abs(mean): tmean = attention(tmean)
rms = c.rms()
trms = '%+.5g' % rms
if prec_TIGHT < abs(rms): trms = attention(trms)
vmin = c.min()
tmin = '%+.5g' % vmin
if prec_TIGHT < abs(vmin): tmin = attention(tmin)
vmax = c.max()
tmax = '%+.5g' % vmax
if prec_TIGHT < abs(vmax): tmax = attention(tmax)
row = n, '%d' % c.nEntries(), tmean, trms, tmin, tmax
rows.append(row)
table = T.table(rows,
title='Test of Carlson forms, Local vs plain integration',
prefix='# ',
alignment='lrllll')
logger.info('Test Carlson forms, Local vs plain integrtation:\n%s' % table)
def _fit_table_ ( rfit , title = '' , prefix = '' ) :
"""Print <code>TFitResult</code> as a table
"""
from ostap.fitting.utils import fit_status, cov_qual
from ostap.logger.colorized import attention, allright
from ostap.logger.utils import pretty_float, pretty_ve, pretty_2ve, fmt_pretty_ve
header = ( '', 'Unit' , 'Value' )
rows = []
## 0. minimized type
row = "Minimizer Type" , '' , rfit.MinimizerType()
rows.append ( row )
## 0. minimized type
v = rfit.IsValid()
if v : row = "Valid" , '' , 'True'
else : row = "Valid" , '' , attention ( 'False')
rows.append ( row )
## 1. fit status
status = rfit.Status()
if status :
row = attention ( 'Status' ) , '' , attention ( fit_status ( status ) )
rows.append ( row )
else :
row = 'Status' , '' , allright ( fit_status ( status ) )
rows.append ( row )
## 4. covariance status
cq = rfit.CovMatrixStatus()
cn = ''
if -1 == cq : cn = cov_qual ( cq )
elif 3 == cq : cn = allright ( cov_qual ( cq ) )
elif cq in ( 0 , 1 , 2 ) : cn = attention ( cov_qual ( cq ) )
else : cn = cov_qual ( cq )
rows.append ( ( 'Covariance matrix quality' , '' , ' ' + cn ) )
## 3-6. chi2,nDoF,chi2/nDoF,minFCN
chi2 = rfit.Chi2 ()
s , n = pretty_float ( chi2 )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Chi2" , n , ' ' + s ) )
##
ndf = rfit.Ndf()
rows.append ( ( "nDoF" , '' , ' ' + '%d' % ndf ) )
##
c2ndf = rfit.Chi2 () / ndf
s , n = pretty_float ( c2ndf )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Chi2/nDoF" , n , ' ' + s ) )
##
minfcn = rfit.MinFcnValue()
s , n = pretty_float ( minfcn )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Minimal FCN" , n , ' ' + s ) )
## 7.Probability in %[%]
prob = rfit.Prob() / 100
rows.append ( ( "Probability" , '[%]' , ' %5.3e' % prob ) )
## 8. distrance to minimum
edm = rfit.Edm()
s , n = pretty_float ( edm )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Estimated distance to minimum" , n , ' ' + s ) )
ncalls = rfit.NCalls()
rows.append ( ( "FCN calls" , '' , ' ' + '%d' % ncalls ) )
##
has_minos = False
for i in rfit :
if not rfit.HasMinosError( i ) : continue
has_minos = True
break
if has_minos :
rows = [ row + ('','','') for row in rows ]
header = header + ( 'neg-minos' , 'pos-minos' , 'Global corr.' )
else :
rows = [ row + ('',) for row in rows ]
header = header + ( 'Global corr.' , )
for i in rfit :
pname = rfit.GetParameterName ( i )
value = rfit.Value ( i )
fixed = rfit.IsParameterFixed ( i )
fmte = ''
if fixed :
v = value
s , n = pretty_float ( v )
s = s + '(fixed)'
nv = n
else :
error = rfit.Error ( i )
v = VE ( value , error * error )
##
fmt , fmtv , fmte , n = fmt_pretty_ve ( v )
s = fmt % ( value / 10**n , error / 10**n )
nv = n
if n : n = '[10^%+d]' % n
else : n = ''
pname = "%-2d: %s"% ( i , pname )
row = pname , n , ' ' + s
if not fixed and rfit.HasMinosError( i ) :
if fmte :
error_low = fmte % ( rfit.LowerError ( i ) / 10**nv )
error_up = fmte % ( rfit.UpperError ( i ) / 10**nv )
else :
error_low = "%+8g" % ( rfit.LowerError ( i ) / 10**nv )
error_up = "%+8g" % ( rfit.UpperError ( i ) / 10**nv )
else :
error_low = ''
error_up = ''
if has_minos :
row = row + ( error_low , error_up )
gcc = rfit.GlobalCC ( i ) * 100
gcc = '%+5.1f%%' % gcc
row = row + ( gcc, )
rows.append ( row )
if not title : title = rfit.GetTitle()
import ostap.logger.table as T
rows = [ header ] + rows
return T.table ( rows , title = title , prefix = prefix )
def _rfr_table_(r, title='', prefix=''):
""" print RooFitResult as a table
>>> result = ...
>>> result.table()
"""
from ostap.fitting.utils import fit_status, cov_qual
rows = []
if r.status():
row = attention(' Status'), '', attention(fit_status(r.status())), ''
rows.append(row)
s, n = pretty_float(r.minNll())
if n: n = '[10^%+d]' % n
else: n = ''
rows.append(("Minimized FCN/NLL value", n, ' ' + s, ''))
s, n = pretty_float(r.edm())
if n: n = '[10^%+d]' % n
else: n = ''
rows.append(('Estimated distance to minimum', n, ' ' + s, ''))
cq = r.covQual()
cn = ''
if -1 == cq:
cn = cov_qual(cq)
elif 3 == cq:
cn = allright(cov_qual(cq))
elif cq in (0, 1, 2):
cn = attention(cov_qual(cq))
else:
cn = cov_qual(cq)
rows.append(('Covariance matrix quality', '', ' ' + cn, ''))
for i in range(r.numStatusHistory()):
label = r.statusLabelHistory(i)
code = r.statusCodeHistory(i)
row = 'Status: %s ' % label, '', '%d' % code
if not code in (0, -1):
row = attention(row[0]), row[1], ' ' + attention(row[2]), ''
else:
row = row[0], row[1], ' ' + allright(row[2]), ''
rows.append(row)
nbadnll = r.numInvalidNLL()
if 0 < nbadnll:
rows.append(('Invalid FCN/NLL evaluations', '', ' %d' % nbadnll, ''))
rows = [('', 'Unit', 'Value', 'Global/max correlation')] + rows
pars_all = r.params(float_only=False)
pars_float = r.params(float_only=True)
## constant/fix parameters
crows = []
for p in pars_all:
if p in pars_float: continue
v, a = pars_all[p]
s, n = pretty_float(v.value())
if n: n = '[10^%+d]' % n
else: n = ''
row = p, n, ' ' + s, ''
crows.append(row)
## floating parameters
frows = []
for p in pars_float:
v, a = pars_float[p]
if not a.hasAsymError():
s, n = pretty_ve(v)
else:
s, n = pretty_2ve(a.getVal(), a.getAsymErrorHi(),
a.getAsymErrorLo())
if n: n = '[10^%+d]' % n
else: n = ''
cc = 'Not available'
if 0 <= cq:
mxr, mxv = r.max_cor(p)
gc = r.globalCorr(p)
cc = '%+5.3f/(%+5.3f,%s)' % (gc, mxr, mxv)
if 0.95 < abs(gc) or 0.95 < abs(mxr): cc = attention(cc)
row = p, n, s, cc
frows.append(row)
crows.sort()
frows.sort()
all = rows + crows + frows
import ostap.logger.table as T
all = T.align_column(all, 0, 'left')
all = T.align_column(all, 1, 'left')
all = T.align_column(all, 2, 'left')
all = T.align_column(all, 3, 'left')
for l in range(len(rows), len(all)):
line = all[l]
line = list(line)
line[0] = allright(line[0])
all[l] = tuple(line)
if title:
return T.table(all, title=title, prefix=prefix)
else:
return T.table(all, title=r.GetTitle(), prefix=prefix)
def test_fitting_fill_1():
## if 1 < 2 :
logger = getLogger('test_fitting_fill_1')
## prepare data
with timing("Prepare test data", logger=logger):
files = prepare_data(4, 5000)
data = Data('S', files)
chain = data.chain
mJPsi = ROOT.RooRealVar('mJPsi', 'mass(J/Psi) [GeV]', 3.0 * GeV, 3.2 * GeV)
# =========================================================================
logger.info(attention('All trivial variables'))
# =========================================================================
variables = [
Variable(mJPsi, accessor='mass'),
Variable('massMeV', 'mass in MeV', 3000, 3200, 'mass*1000.0'),
Variable('vv102', 'vv10[2]', -1, 100, '1.0*vv10[2]'),
Variable('fevt', accessor='1.0*evt'), ('pt', ), ('eta', ),
('x', 'some variable', 0, 5000, '(mass+pt+eta)/eta')
]
config = {'variables': variables, 'selection': "pt>7 && eta<3"}
with timing("No SHORTCUT, no FRAME", logger=None) as t1:
logger.info(attention(t1.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=False, use_frame=False)
ds1_1 = selector.data
with timing(" SHORTCUT, no FRAME", logger=None) as t2:
logger.info(attention(t2.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=True, use_frame=False)
ds1_2 = selector.data
with timing("No SHORTCUT, FRAME", logger=None) as t3:
logger.info(attention(t3.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=False, use_frame=True)
ds1_3 = selector.data
with timing(" SHORTCUT, FRAME", logger=None) as t4:
logger.info(attention(t4.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=True, use_frame=True)
ds1_4 = selector.data
if DataSet_NEW_FILL:
with timing(" pure-FRAME (new) ", logger=None) as t5:
logger.info(attention(t5.name))
ds1_5, _ = chain.make_dataset(silent=False, **config)
table = [('Configuration', 'CPU')]
table.append((t1.name, '%.3fs' % t1.delta))
table.append((t2.name, '%.3fs' % t2.delta))
table.append((t3.name, '%.3fs' % t3.delta))
table.append((t4.name, '%.3fs' % t4.delta))
if DataSet_NEW_FILL:
table.append((t5.name, '%.3fs' % t5.delta))
title1 = "All trivial variables"
table1 = T.table(table, title=title1, prefix='# ', alignment='rr')
logger.info('%s\n%s' % (title1, table1))
if ds1_1 != ds1_2:
logger.error('Datasets ds1_1 and ds1_2 are different!')
if ds1_1 != ds1_3:
logger.error('Datasets ds1_1 and ds1_3 are different!')
if ds1_1 != ds1_4:
logger.error('Datasets ds1_1 and ds1_4 are different!')
if DataSet_NEW_FILL:
if ds1_1 != ds1_5:
logger.error('Datasets ds1_1 and ds1_5 are different!')
with timing("No SHORTCUT, no FRAME", logger=None) as t1:
logger.info(attention(t1.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=False,
use_frame=False,
max_files=1)
ds1p_1 = selector.data
with timing(" SHORTCUT, no FRAME", logger=None) as t2:
logger.info(attention(t2.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=True,
use_frame=False,
max_files=1)
ds1p_2 = selector.data
with timing("No SHORTCUT, FRAME", logger=None) as t3:
logger.info(attention(t3.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=False,
use_frame=True,
max_files=1)
ds1p_3 = selector.data
with timing(" SHORTCUT, FRAME", logger=None) as t4:
logger.info(attention(t4.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=True,
use_frame=True,
max_files=1)
ds1p_4 = selector.data
table = [('Configuration', 'CPU')]
table.append((t1.name, '%.3fs' % t1.delta))
table.append((t2.name, '%.3fs' % t2.delta))
table.append((t3.name, '%.3fs' % t3.delta))
table.append((t4.name, '%.3fs' % t4.delta))
title1p = "All trivial variables (parallel)"
table1p = T.table(table, title=title1p, prefix='# ', alignment='rr')
logger.info('%s\n%s' % (title1p, table1p))
if ds1_1 != ds1p_1:
logger.error('Datasets ds1_1 and ds1p_1 are different!')
if ds1_2 != ds1p_2:
logger.error('Datasets ds1_2 and ds1p_2 are different!')
if ds1_3 != ds1p_3:
logger.error('Datasets ds1_3 and ds1p_3 are different!')
if ds1_4 != ds1p_4:
logger.error('Datasets ds1_4 and ds1p_4 are different!')
# =========================================================================
logger.info(attention('Trivial variables + CUT'))
# =========================================================================
variables = [
Variable(mJPsi, accessor='mass'),
Variable('massMeV', 'mass in MeV', 3000, 3200, 'mass*1000'),
Variable('vv102', 'vv10[2]', -1, 100, '1.0*vv10[2]'),
Variable('fevt', accessor='1.0*evt'), ('pt', ), ('eta', ),
('x', 'some variable', 0, 5000, '(mass+pt+eta)/eta')
]
if not DILL_PY3_issue:
config = {
'variables': variables,
'selection': "pt>7 && eta<3",
'cuts': lambda s: s.pt > 3
} ## ATTENTION: no trivial cuts!
else:
logger.warning('There is an issue with dill+python3: avoid lambda!')
config = {
'variables': variables,
'selection': "pt>7 && eta<3",
'cuts': ptcut
} ## ATTENTION: no trivial cuts!
with timing("No SHORTCUT, no FRAME", logger=None) as t1:
logger.info(attention(t1.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=False, use_frame=False)
ds2_1 = selector.data
with timing(" SHORTCUT, no FRAME", logger=None) as t2:
logger.info(attention(t2.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=True, use_frame=False)
ds2_2 = selector.data
with timing("No SHORTCUT, FRAME", logger=None) as t3:
logger.info(attention(t3.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=False, use_frame=True)
ds2_3 = selector.data
with timing(" SHORTCUT, FRAME", logger=None) as t4:
logger.info(attention(t4.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=True, use_frame=True)
ds2_4 = selector.data
table = [('Configuration', 'CPU')]
table.append((t1.name, '%.3fs' % t1.delta))
table.append((t2.name, '%.3fs' % t2.delta))
table.append((t3.name, '%.3fs' % t3.delta))
table.append((t4.name, '%.3fs' % t4.delta))
title2 = "Trivial variables + CUT"
table2 = T.table(table, title=title2, prefix='# ', alignment='rr')
logger.info('%s\n%s' % (title2, table2))
if ds2_1 != ds2_2:
logger.error('Datasets ds2_1 and ds2_2 are different!')
if ds2_1 != ds2_3:
logger.error('Datasets ds2_1 and ds2_3 are different!')
if ds2_1 != ds2_4:
logger.error('Datasets ds2_1 and ds2_4 are different!')
with timing("No SHORTCUT, no FRAME", logger=None) as t1:
logger.info(attention(t1.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=False,
use_frame=False,
maX_files=1)
ds2p_1 = selector.data
with timing(" SHORTCUT, no FRAME", logger=None) as t2:
logger.info(attention(t2.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=True,
use_frame=False,
maX_files=1)
ds2p_2 = selector.data
with timing("No SHORTCUT, FRAME", logger=None) as t3:
logger.info(attention(t3.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=False,
use_frame=True,
max_files=1)
ds2p_3 = selector.data
with timing(" SHORTCUT, FRAME", logger=None) as t4:
logger.info(attention(t4.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=True,
use_frame=True,
max_files=1)
ds2p_4 = selector.data
table = [('Configuration', 'CPU')]
table.append((t1.name, '%.3fs' % t1.delta))
table.append((t2.name, '%.3fs' % t2.delta))
table.append((t3.name, '%.3fs' % t3.delta))
table.append((t4.name, '%.3fs' % t4.delta))
title2p = "Trivial variables + CUT (parallel)"
table2p = T.table(table, title=title2p, prefix='# ', alignment='rr')
logger.info('%s\n%s' % (title2p, table2p))
if ds1_1 != ds2_1:
logger.error('Datasets ds1_1 and ds2_1 are different!')
if ds2_1 != ds2p_1:
logger.error('Datasets ds2_1 and ds2p_1 are different!')
if ds2_2 != ds2p_2:
logger.error('Datasets ds2_2 and ds2p_2 are different!')
if ds2_3 != ds2p_3:
logger.error('Datasets ds2_3 and ds2p_3 are different!')
if ds2_4 != ds2p_4:
logger.error('Datasets ds2_4 and ds2p_4 are different!')
# =========================================================================
logger.info(attention('Non-trivial variables'))
# =========================================================================
if not DILL_PY3_issue:
variables = [
Variable(mJPsi, accessor='mass'),
Variable('massMeV', 'mass in MeV', 3000, 3200, 'mass*1000'),
Variable('vv102', 'vv10[2]', -1, 100, '1.0*vv10[2]'),
Variable('fevt', accessor='1.0*evt'), ('pt', ), ('eta', ),
('x', 'some variable', 0, 5000, lambda s:
(s.mass + s.pt + s.eta) / s.eta)
]
else:
logger.warning('There is an issue with dill+python3: avoid lambda!')
variables = [
Variable(mJPsi, accessor='mass'),
Variable('massMeV', 'mass in MeV', 3000, 3200, 'mass*1000'),
Variable('vv102', 'vv10[2]', -1, 100, '1.0*vv10[2]'),
Variable('fevt', accessor='1.0*evt'), ('pt', ), ('eta', ),
('x', 'some variable', 0, 5000, xvar)
]
config = {'variables': variables, 'selection': "pt>7 && eta<3"}
with timing("No SHORTCUT, no FRAME", logger=None) as t1:
logger.info(attention(t1.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=False, use_frame=False)
ds3_1 = selector.data
with timing(" SHORTCUT, no FRAME", logger=None) as t2:
logger.info(attention(t2.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=True, use_frame=False)
ds3_2 = selector.data
with timing("No SHORTCUT, FRAME", logger=None) as t3:
logger.info(attention(t3.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=False, use_frame=True)
ds3_3 = selector.data
with timing(" SHORTCUT, FRAME", logger=None) as t4:
logger.info(attention(t4.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=True, use_frame=True)
ds3_4 = selector.data
table = [('Configuration', 'CPU')]
table.append((t1.name, '%.3fs' % t1.delta))
table.append((t2.name, '%.3fs' % t2.delta))
table.append((t3.name, '%.3fs' % t3.delta))
table.append((t4.name, '%.3fs' % t4.delta))
title3 = "Non-trivial variables"
table3 = T.table(table, title=title3, prefix='# ', alignment='rr')
logger.info('%s\n%s' % (title3, table3))
if ds1_1 != ds3_1:
logger.error('Datasets ds1_1 and ds3_1 are different!')
if ds3_1 != ds3_2:
logger.error('Datasets ds3_1 and ds2_2 are different!')
if ds3_1 != ds3_3:
logger.error('Datasets ds3_1 and ds2_3 are different!')
if ds3_1 != ds3_4:
logger.error('Datasets ds3_1 and ds2_4 are different!')
with timing("No SHORTCUT, no FRAME", logger=None) as t1:
logger.info(attention(t1.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=False,
use_frame=False,
max_files=1)
ds3p_1 = selector.data
with timing(" SHORTCUT, no FRAME", logger=None) as t2:
logger.info(attention(t2.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=True,
use_frame=False,
max_files=1)
ds3p_2 = selector.data
with timing("No SHORTCUT, FRAME", logger=None) as t3:
logger.info(attention(t3.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=False,
use_frame=True,
max_files=1)
ds3p_3 = selector.data
with timing(" SHORTCUT, FRAME", logger=None) as t4:
logger.info(attention(t4.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=True,
use_frame=True,
max_files=1)
ds3p_4 = selector.data
table = [('Configuration', 'CPU')]
table.append((t1.name, '%.3fs' % t1.delta))
table.append((t2.name, '%.3fs' % t2.delta))
table.append((t3.name, '%.3fs' % t3.delta))
table.append((t4.name, '%.3fs' % t4.delta))
title3p = "Non-trivial variables (parallel)"
table3p = T.table(table, title=title3p, prefix='# ', alignment='rr')
logger.info('%s\n%s' % (title3p, table3p))
if ds3_1 != ds3p_1:
logger.error('Datasets ds3_1 and ds3p_1 are different!')
if ds3_2 != ds3p_2:
logger.error('Datasets ds3_2 and ds3p_2 are different!')
if ds3_3 != ds3p_3:
logger.error('Datasets ds3_3 and ds3p_3 are different!')
if ds3_4 != ds3p_4:
logger.error('Datasets ds3_4 and ds3p_4 are different!')
# =========================================================================
logger.info(attention('Non-trivial variables + CUT'))
# =========================================================================
if not DILL_PY3_issue:
variables = [
Variable(mJPsi, accessor='mass'),
Variable('massMeV', 'mass in MeV', 3000, 3200, 'mass*1000'),
Variable('vv102', 'vv10[2]', -1, 100, '1.0*vv10[2]'),
Variable('fevt', accessor='1.0*evt'), ('pt', ), ('eta', ),
('x', 'some variable', 0, 5000, lambda s:
(s.mass + s.pt + s.eta) / s.eta)
]
else:
logger.warning('There is an issue with dill+python3: avoid lambda!')
variables = [
Variable(mJPsi, accessor='mass'),
Variable('massMeV', 'mass in MeV', 3000, 3200, 'mass*1000'),
Variable('vv102', 'vv10[2]', -1, 100, '1.0*vv10[2]'),
Variable('fevt', accessor='1.0*evt'), ('pt', ), ('eta', ),
('x', 'some variable', 0, 5000, xvar)
]
if not DILL_PY3_issue:
config = {
'variables': variables,
'selection': "pt>7 && eta<3",
'cuts': lambda s: s.pt > 3
} ## ATTENTION: no trivial cuts!
else:
logger.warning('There is an issue with dill+python3: avoid lambda!')
config = {
'variables': variables,
'selection': "pt>7 && eta<3",
'cuts': ptcut
} ## ATTENTION: no trivial cuts!
with timing("No SHORTCUT, no FRAME", logger=None) as t1:
logger.info(attention(t1.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=False, use_frame=False)
ds4_1 = selector.data
with timing(" SHORTCUT, no FRAME", logger=None) as t2:
logger.info(attention(t2.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=True, use_frame=False)
ds4_2 = selector.data
with timing("No SHORTCUT, FRAME", logger=None) as t3:
logger.info(attention(t3.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=False, use_frame=True)
ds4_3 = selector.data
with timing(" SHORTCUT, FRAME", logger=None) as t4:
logger.info(attention(t4.name))
selector = SelectorWithVars(**config)
chain.fill_dataset(selector, shortcut=True, use_frame=True)
ds4_4 = selector.data
table = [('Configuration', 'CPU')]
table.append((t1.name, '%.3fs' % t1.delta))
table.append((t2.name, '%.3fs' % t2.delta))
table.append((t3.name, '%.3fs' % t3.delta))
table.append((t4.name, '%.3fs' % t4.delta))
title4 = "Non-trivial variables + CUT"
table4 = T.table(table, title=title4, prefix='# ', alignment='rr')
logger.info('%s\n%s' % (title4, table4))
if ds1_1 != ds4_1:
logger.error('Datasets ds1_1 and ds4_1 are different!')
if ds4_1 != ds4_2:
logger.error('Datasets ds4_1 and ds4_2 are different!')
if ds4_1 != ds4_3:
logger.error('Datasets ds4_1 and ds4_3 are different!')
if ds4_1 != ds4_4:
logger.error('Datasets ds4_1 and ds4_4 are different!')
with timing("No SHORTCUT, no FRAME", logger=None) as t1:
logger.info(attention(t1.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=False,
use_frame=False,
max_files=1)
ds4p_1 = selector.data
with timing(" SHORTCUT, no FRAME", logger=None) as t2:
logger.info(attention(t2.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=True,
use_frame=False,
max_files=1)
ds4p_2 = selector.data
with timing("No SHORTCUT, FRAME", logger=None) as t3:
logger.info(attention(t3.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=False,
use_frame=True,
max_files=1)
ds4p_3 = selector.data
with timing(" SHORTCUT, FRAME", logger=None) as t4:
logger.info(attention(t4.name))
selector = SelectorWithVars(**config)
chain.parallel_fill(selector,
shortcut=True,
use_frame=True,
max_files=1)
ds4p_4 = selector.data
table = [('Configuration', 'CPU')]
table.append((t1.name, '%.3fs' % t1.delta))
table.append((t2.name, '%.3fs' % t2.delta))
table.append((t3.name, '%.3fs' % t3.delta))
table.append((t4.name, '%.3fs' % t4.delta))
title4p = "Non-trivial variables + CUT (parallel)"
table4p = T.table(table, title=title4p, prefix='# ', alignment='rr')
logger.info('%s\n%s' % (title4p, table4p))
if ds4_1 != ds4p_1:
logger.error('Datasets ds4_1 and ds4p_1 are different!')
if ds4_2 != ds4p_2:
logger.error('Datasets ds4_2 and ds4p_2 are different!')
if ds4_3 != ds4p_3:
logger.error('Datasets ds4_3 and ds4p_3 are different!')
if ds4_4 != ds4p_4:
logger.error('Datasets ds4_4 and ds4p_4 are different!')
logger.info('%s\n%s' % (title1, table1))
logger.info('%s\n%s' % (title1p, table1p))
logger.info('%s\n%s' % (title2, table2))
logger.info('%s\n%s' % (title2p, table2p))
logger.info('%s\n%s' % (title3, table3))
logger.info('%s\n%s' % (title3p, table3p))
logger.info('%s\n%s' % (title4, table4))
logger.info('%s\n%s' % (title4p, table4p))