def frame_progress(self, length, width=None, symbol="#"):
""" Draw (lazy) progress bar for the DataFrame:
>>> f = DataFrame ( ... )
>>> p = f.ProgressBar ( 1000 ) ## number of elements!
>>> ....
"""
cnt = self.Count()
if not isatty(): return cnt
left = "[ "
right = " ]"
minw = len(left) + len(right) + 3 + 1 + 1 + 1
length = length if isinstance(length,
integer_types) and 0 < length else len(self)
width = width if isinstance(
width, integer_types) and 10 + minw <= width else terminal_size()[1]
width = max(10, width - minw)
nchunks = max(250, width + 1)
csize, rr = divmod(length, nchunks)
if rr: nchunks += 1
symbol = allright(symbol)
fun = Ostap.Utils.frame_progress(nchunks, width, symbol, ' ', left, right)
cnt.OnPartialResultSlot(csize, fun)
return cnt
def frame_progress(self, length, width=None, symbol="#"):
""" Draw (lazy) progress bar for the DataFrame:
>>> f = DataFrame ( ... )
>>> p = f.ProgressBar ( 1000 ) ## number of elements!
>>> ....
"""
cnt = self.Count()
if not isatty(): return cnt
length = length if isinstance(length,
integer_types) and 0 < length else len(self)
width = width if isinstance(
width, integer_types) and 10 < width else terminal_size()[1]
if width < 16: width = 16
nchunks = width - 14
csize = max(int(length / nchunks), 1)
left = "[ "
right = " ]"
symbol = allright(symbol)
fun = Ostap.Utils.frame_progress(csize, nchunks, symbol, ' ', left, right)
cnt.OnPartialResultSlot(csize, fun)
return cnt
def table(rows, title='', prefix=''):
"""Format the list of rows as a table.
- Each row is a sequence of column cells.
- The first row defines the column headers.
When terminaltables package is accessible,
use it to get nice pretty tables, otherwise use some
home-made primitive replacement
- see https://pypi.org/project/terminaltables
- see https://github.com/Robpol86/terminaltables
>>> table_data = [
... ( 'Name' , 'Occupation' , 'Note' ) ,
... ( 'Alice' , '?' , '---' ) ,
... ( 'Bob' , 'unemployed' , '' ) ]
>>> t = table ( table_data , 'Title' )
>>> print (t)
"""
from ostap.utils.basic import isatty
title = allright(decolorize(title))
if rows:
rows = list(rows)
header_row = rows[0]
header_row = [infostr(decolorize(c)) for c in header_row]
rows[0] = header_row
rows = tuple(rows)
if terminaltables and isatty():
table_instance = terminaltables.SingleTable(rows, title)
table_instance.justify_columns[0] = 'left'
table_instance.justify_columns[2] = 'right'
return add_prefix(table_instance.table, prefix)
elif terminaltables:
title = allright(title)
table_instance = terminaltables.AsciiTable(rows, title)
table_instance.justify_columns[0] = 'left'
table_instance.justify_columns[2] = 'right'
t = table_instance.table
return add_prefix(table_instance.table, prefix)
## use the local replacement
return the_table(rows, title, prefix)
def table(rows,
title='',
prefix='',
alignment=(),
wrap_width=-1,
indent=wrap_indent):
"""Format the list of rows as a table.
- Each row is a sequence of column cells.
- The first row defines the column headers.
When terminaltables package is accessible,
use it to get nice pretty tables, otherwise use some
home-made primitive replacement
- see https://pypi.org/project/terminaltables
- see https://github.com/Robpol86/terminaltables
>>> table_data = [
... ( 'Name' , 'Occupation' , 'Note' ) ,
... ( 'Alice' , '?' , '---' ) ,
... ( 'Bob' , 'unemployed' , '' ) ]
>>> t = table ( table_data , 'Title' )
>>> print (t)
"""
from ostap.utils.basic import isatty
title = allright(decolorize(title))
if rows:
rows = list(rows)
header_row = rows[0]
header_row = [infostr(decolorize(c)) for c in header_row]
rows[0] = header_row
rows = tuple(rows)
rows = [list(row) for row in rows]
if not terminaltables:
## use the local replacement
return the_table(rows, title, prefix, alignment=alignment)
if isatty():
title = allright(title)
table_instance = terminaltables.SingleTable(rows, title)
else:
title = allright(title)
table_instance = terminaltables.AsciiTable(rows, title)
cw = table_instance.column_widths
nc = len(cw)
wraps = [i for (i, a) in enumerate(alignment) if a in wrapped]
if wraps:
from terminaltables.width_and_alignment import max_dimensions
widths = max_dimensions(table_instance.table_data,
table_instance.padding_left,
table_instance.padding_right)[2]
widths = sum(l for (i, l) in enumerate(widths) if not i in wraps)
widths += nc + 1 + len(prefix) + 4 + 2 * len(wraps)
_, w = terminal_size()
ww = w - widths
ww, _ = divmod(ww, len(wraps))
if 12 < ww and ww < wrap_width: wrap_width = ww
elif 12 < ww and wrap_width <= 0: wrap_width = ww
if wrap_width < 12: wrap_width = max_width
nw = len(wraps)
for i, a in zip(range(nc), alignment):
if a and isinstance(a, str):
al = a.lower()
if al in left: table_instance.justify_columns[i] = 'left'
elif al in right: table_instance.justify_columns[i] = 'right'
elif al in center: table_instance.justify_columns[i] = 'center'
elif al in wrapped:
maxw = table_instance.column_max_width(i)
if 15 < wrap_width * nw < maxw:
maxw = (wrap_width - 3) * nw if 1 < nw else wrap_width
if maxw < 15:
maxw = (wrap_width - 3) * nw if 1 < nw else wrap_width
if maxw < 15:
maxw = (max_width - 3) * nw if 1 < nw else max_width
width = maxw / nw if 1 < nw else maxw
for l, line in enumerate(table_instance.table_data):
if width < len(line[i]):
table_instance.table_data[l][i] = textwrap.fill(
indent + line[i], wrap_width)
return add_prefix(table_instance.table, prefix)
def the_table(rows, title='', prefix=''):
"""Format the list of rows as a table (home-made primitive)
- Each row is a sequence of column cells.
- The first row defines the column headers.
>>> table_data = [
... ( 'Name' , 'Occupation' , 'Note' ) ,
... ( 'Alice' , '?' , '---' ) ,
... ( 'Bob' , 'unemployed' , '' ) ]
>>> t = the_table ( table_data , 'Title' )
>>> print (t)
"""
## calculate the number of columns
nc = 0
for row in rows:
nc = max(nc, len(row))
## calculate the maximum width for columns
widths = {}
for row in rows:
cols = [c for c in row]
while len(cols) < nc:
cols.append('')
for i, c in enumerate(cols):
if not i in widths: widths[i] = 1
widths[i] = max(widths[i], len(decolorize(c)))
totwidth = 0
for c in widths:
totwidth += widths[c]
totwidth += (nc - 1) * 3
if totwidth < len(title):
delta = 1 + (len(title) - totwidth) // nc
for c in widths:
widths[c] += delta
hformats = ["{:^%d}" % widths[c] for c in range(nc)]
rformats = [" {:^%d} " % widths[c] for c in range(nc)]
# the first column is left adjusted
rformats[0] = rformats[0].replace('^', '<')
# the last column is right adjusted
rformats[-1] = rformats[-1].replace('^', '>')
seps = ['-' * (widths[c] + 2) for c in range(nc)]
sepline = '+' + "+".join(seps) + '+'
table = []
if title:
sline = '+' + '-' * (len(sepline) - 2) + '+'
tfmt = "{:^%d}" % (len(sepline) - 4)
t = '| ' + allright(tfmt.format(decolorize(title))) + ' |'
table.append(sline)
table.append(t)
table.append(sepline)
for i, row in enumerate(rows):
cols = [c for c in row]
while len(cols) < nc:
cols.append('')
if 0 == i:
table.append('| ' + ' | '.join([
infostr(f.format(decolorize(i)))
for (f, i) in zip(hformats, cols)
]) + ' |')
table.append(sepline)
else:
table.append('|' + '|'.join(
[f.format(decolorize(i))
for (f, i) in zip(rformats, cols)]) + '|')
table.append(sepline)
return prefix + ('\n' + prefix).join(table) if prefix else '\n'.join(table)
def _rfr_table_ ( r , title = '' , prefix = '' , more_vars = {} ) :
""" print RooFitResult as a table
>>> result = ...
>>> result.table()
"""
from ostap.fitting.utils import fit_status, cov_qual
rows = []
## 1. fit status
status = r.status()
if status :
row = attention ( 'Status' ) , '' , attention ( fit_status ( status ) ) , ''
rows.append ( row )
else :
row = 'Status' , '' , allright ( fit_status ( status ) ) , ''
rows.append ( row )
## 2. minumum NLL
s , n = pretty_float ( r.minNll() )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Minimized FCN/NLL value" , n , ' ' + s , '' ) )
s , n = pretty_float ( r.edm () )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( 'Estimated distance to minimum' , n , ' ' + s , '' ) )
cq = r.covQual()
cn = ''
if -1 == cq :
cn = cov_qual ( cq )
elif 3 == cq :
cn = allright ( cov_qual ( cq ) )
elif cq in ( 0 , 1 , 2 ) :
cn = attention ( cov_qual ( cq ) )
else :
cn = cov_qual ( cq )
rows.append ( ( 'Covariance matrix quality' , '' , ' ' + cn , '' ) )
for i in range ( r.numStatusHistory() ) :
label = r.statusLabelHistory ( i )
code = r.statusCodeHistory ( i )
row = 'Status: %s '% label , '' , '%d' % code
if not code in ( 0 , -1 ) :
row = attention ( row [ 0 ] ) , row [ 1 ] , ' ' + attention ( row [ 2 ] ) , ''
else :
row = row [ 0 ] , row [ 1 ] , ' ' + allright ( row [ 2 ] ) , ''
rows.append ( row )
nbadnll = r.numInvalidNLL()
if 0 < nbadnll :
rows.append ( ( 'Invalid FCN/NLL evaluations' , '' , ' %d' % nbadnll , '' ) )
with_globcorr = not ( (6,24) <= root_info < (6,28) )
if with_globcorr : rows = [ ( '', 'Unit', 'Value' , 'Global/max correlation [%]') ] + rows
else : rows = [ ( '', 'Unit', 'Value' , 'Max correlation [%]') ] + rows
pars_all = r.params ( float_only = False )
pars_float = r.params ( float_only = True )
## constant/fix parameters
crows = []
for p in pars_all :
if p in pars_float : continue
v , a = pars_all [ p ]
s , n = pretty_float ( v.value() )
if n : n = '[10^%+d]' % n
else : n = ''
row = p , n , ' ' + s + ' (fix)' , ''
crows.append ( row )
## floating parameters
max_corr = False
frows = []
for p in pars_float :
v , a = pars_float [ p ]
if not a.hasAsymError() :
s , n = pretty_ve ( v )
else :
s , n = pretty_2ve ( a.getVal() , a.getAsymErrorHi() , a.getAsymErrorLo() )
if n : n = '[10^%+d]' % n
else : n = ''
if 0 <= cq and 1 < len ( pars_float ) :
mxr , mxv = r.max_cor ( p )
if with_globcorr :
gc = -1.0
gc = r.globalCorr ( p ) if 3 == cq else -1.00
if 0 <= gc : cc = '% +5.1f/(% +5.1f,%s)' % ( gc*100 , mxr*100 , mxv )
else : cc = '% +5.1f : %-s' % ( mxr * 100 , mxv )
if 0.95 < abs ( mxr ) or 0.95 < gc : cc = attention ( cc )
else :
cc = '% +5.1f : %-s' % ( mxr * 100 , mxv )
if 0.95 < abs ( mxr ) : cc = attention ( cc )
max_corr = True
row = p , n , s , cc
else :
row = p , n , s
frows.append ( row )
## more parameters
mrows = []
for p in sorted ( more_vars ) :
func = more_vars [ p ]
v = func ( r )
s , n = pretty_ve ( v )
if n : n = '[10^%+d]' % n
else : n = ''
cc = 'derived'
row = p , n , s , cc
mrows.append ( row )
crows.sort()
frows.sort()
all = rows + crows + frows + mrows
import ostap.logger.table as T
return T.table ( all , title = title if title else r.GetTitle() , prefix = prefix , alignment = 'llll' )
def makeWeights(
dataset,
plots=[],
database="weights.db",
compare=None, ## comparison function
delta=0.01, ## delta for ``mean'' weight variation
minmax=0.03, ## delta for ``minmax'' weight variation
power=None, ## auto-determination
debug=True, ## save intermediate information in DB
make_plots=False, ## make plots
tag="Reweighting"):
"""The main function: perform one re-weighting iteration
and reweight ``MC''-data set to looks as ``data''(reference) dataset
>>> results = makeWeights (
... dataset , ## data source to be reweighted (DataSet, TTree, abstract source)
... plots , ## reweighting plots
... database , ## datadabse to store/update reweigting results
... delta , ## stopping criteria for `mean` weight variation
... minmax , ## stopping criteria for `min/max` weight variation
... power , ## effective power to apply to the weigths
... debug = True , ## store debuig information in database
... make_plots = True , ## produce useful comparison plots
... tag = 'RW' ) ## tag for better printout
If `make_plots = False`, it returns the tuple of active reweitings:
>>> active = makeWeights ( ... , make_plots = False , ... )
Otherwise it also returns list of comparison plots
>>> active, cmp_plots = makeWeights ( ... , make_plots = True , ... )
>>> for item in cmp_plots :
... what = item.what
... hdata = item.data
... hmc = item.mc
... hweight = item.weight
If no more rewighting iteratios required, <code>active</code> is an empty tuple
"""
assert 0 < delta, "makeWeights(%s): Invalid value for ``delta'' %s" % (
tag, delta)
assert 0 < minmax, "makeWeights(%s): Invalid value for ``minmax'' %s" % (
tag, minmax)
from ostap.logger.colorized import allright, attention, infostr
from ostap.utils.basic import isatty
nplots = len(plots)
## if 1 < nplots :
## import math
## fudge_factor = math.sqrt ( 1.0 / max ( 2.0 , nplots - 1.0 ) )
## delta = delta * fudge_factor
## minmax = minmax * fudge_factor
## list of plots to compare
cmp_plots = []
## reweighting summary table
header = ('Reweighting', 'wmin/wmax', 'OK?', 'wrms[%]', 'OK?', 'chi2/ndf',
'ww', 'exp')
rows = {}
save_to_db = []
## number of active plots for reweighting
for wplot in plots:
what = wplot.what ## variable/function to plot/compare
how = wplot.how ## weight and/or additional cuts
address = wplot.address ## address in database
hdata0 = wplot.data ## original "DATA" object
hmc0 = wplot.mc_histo ## original "MC" histogram
ww = wplot.w ## relative weight
projector = wplot.projector ## projector for MC data
ignore = wplot.ignore ## ignore for weigtht building?
#
# normalize the data
#
hdata = hdata0
if isinstance(hdata, ROOT.TH1): hdata = hdata.density()
# =====================================================================
## make a plot on (MC) data with the weight
# =====================================================================
hmc0 = projector(dataset, hmc0, what, how)
st = hmc0.stat()
mnmx = st.minmax()
if iszero(mnmx[0]):
logger.warning("%s: statistic goes to zero %s/``%s''" %
(tag, st, address))
elif mnmx[0] <= 0:
logger.warning("%s: statistic is negative %s/``%s''" %
(tag, st, address))
# =====================================================================
## normalize MC
# =====================================================================
hmc = hmc0.density()
# =====================================================================
## calculate the reweighting factor : a bit conservative (?)
# this is the only important line
# =====================================================================
# try to exploit finer binning if/when possible
hboth = isinstance(hmc, ROOT.TH1) and isinstance(hdata, ROOT.TH1)
if hboth and 1 == hmc.dim () and 1 == hdata.dim () and \
len ( hmc ) >= len( hdata ) :
w = (1.0 / hmc) * hdata ## NB!
elif hboth and 2 == hmc.dim () and 2 == hdata.dim () and \
( hmc.binsx() >= hdata.binsx() ) and \
( hmc.binsy() >= hdata.binsy() ) :
w = (1.0 / hmc) * hdata ## NB!
elif hboth and 3 == hmc.dim () and 3 == hdata.dim () and \
( hmc.binsx() >= hdata.binsx() ) and \
( hmc.binsy() >= hdata.binsy() ) and \
( hmc.binsz() >= hdata.binsz() ) :
w = (1.0 / hmc) * hdata ## NB!
else:
w = hdata / hmc ## NB!
# =====================================================================
## scale & get the statistics of weights
w /= w.stat().mean().value()
cnt = w.stat()
#
mnw, mxw = cnt.minmax()
wvar = cnt.rms() / cnt.mean()
good1 = wvar.value() <= delta
good2 = abs(mxw - mnw) <= minmax
good = good1 and good2 ## small variance?
#
c2ndf = 0
for i in w:
c2ndf += w[i].chi2(1.0)
c2ndf /= (len(w) - 1)
## build the row in the summary table
row = address , \
'%-5.3f/%5.3f' % ( cnt.minmax()[0] , cnt.minmax()[1] ) , \
allright ( '+' ) if good2 else attention ( '-' ) , \
(wvar * 100).toString('%6.2f+-%-6.2f') , \
allright ( '+' ) if good1 else attention ( '-' ) , '%6.2f' % c2ndf
## make plots at the start of each iteration?
if make_plots:
item = ComparisonPlot(what, hdata, hmc, w)
cmp_plots.append(item)
row = tuple(list(row) + ['%4.3f' % ww if 1 != ww else ''])
rows[address] = row
#
## make decision based on the variance of weights
#
mnw, mxw = cnt.minmax()
if (not good) and (not ignore): ## small variance?
save_to_db.append((address, ww, hdata0, hmc0, hdata, hmc, w))
# =====================================================================
## make a comparison (if needed)
# =====================================================================
if compare: compare(hdata0, hmc0, address)
active = tuple([p[0] for p in save_to_db])
nactive = len(active)
if power and callable(power):
eff_exp = power(nactive)
elif isinstance(power, num_types) and 0 < power <= 1.5:
eff_exp = 1.0 * power
elif 1 == nactive and 1 < len(plots):
eff_exp = 0.95
elif 1 == nactive:
eff_exp = 1.00
else:
eff_exp = 1.10 / max(nactive, 1)
while database and save_to_db:
entry = save_to_db.pop()
address, ww, hd0, hm0, hd, hm, weight = entry
cnt = weight.stat()
mnw, mxw = cnt.minmax()
## avoid too large or too small weights
for i in weight:
w = weight[i]
if w.value() < 0.5:
weight[i] = VE(0.5, w.cov2())
elif w.value() > 2.0:
weight[i] = VE(2.0, w.cov2())
if 1 < nactive and 1 != ww:
eff_exp *= ww
logger.info("%s: apply ``effective exponent'' of %.3f for ``%s''" %
(tag, eff_exp, address))
if 1 != eff_exp and 0 < eff_exp:
weight = weight**eff_exp
row = list(rows[address])
row.append('%4.3f' % eff_exp)
rows[address] = tuple(row)
with DBASE.open(database) as db:
db[address] = db.get(address, []) + [weight]
if debug:
addr = address + ':REWEIGHTING'
db[addr] = db.get(addr, []) + list(entry[2:])
del hd0, hm0, hd, hm, weight, entry
table = [header]
for row in rows:
table.append(rows[row])
import ostap.logger.table as Table
logger.info(
'%s, active:#%d \n%s ' %
(tag, nactive,
Table.table(table, title=tag, prefix='# ', alignment='lccccccc')))
cmp_plots = tuple(cmp_plots)
return (active, cmp_plots) if make_plots else active
def _fit_table_ ( rfit , title = '' , prefix = '' ) :
"""Print <code>TFitResult</code> as a table
"""
from ostap.fitting.utils import fit_status, cov_qual
from ostap.logger.colorized import attention, allright
from ostap.logger.utils import pretty_float, pretty_ve, pretty_2ve, fmt_pretty_ve
header = ( '', 'Unit' , 'Value' )
rows = []
## 0. minimized type
row = "Minimizer Type" , '' , rfit.MinimizerType()
rows.append ( row )
## 0. minimized type
v = rfit.IsValid()
if v : row = "Valid" , '' , 'True'
else : row = "Valid" , '' , attention ( 'False')
rows.append ( row )
## 1. fit status
status = rfit.Status()
if status :
row = attention ( 'Status' ) , '' , attention ( fit_status ( status ) )
rows.append ( row )
else :
row = 'Status' , '' , allright ( fit_status ( status ) )
rows.append ( row )
## 4. covariance status
cq = rfit.CovMatrixStatus()
cn = ''
if -1 == cq : cn = cov_qual ( cq )
elif 3 == cq : cn = allright ( cov_qual ( cq ) )
elif cq in ( 0 , 1 , 2 ) : cn = attention ( cov_qual ( cq ) )
else : cn = cov_qual ( cq )
rows.append ( ( 'Covariance matrix quality' , '' , ' ' + cn ) )
## 3-6. chi2,nDoF,chi2/nDoF,minFCN
chi2 = rfit.Chi2 ()
s , n = pretty_float ( chi2 )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Chi2" , n , ' ' + s ) )
##
ndf = rfit.Ndf()
rows.append ( ( "nDoF" , '' , ' ' + '%d' % ndf ) )
##
c2ndf = rfit.Chi2 () / ndf
s , n = pretty_float ( c2ndf )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Chi2/nDoF" , n , ' ' + s ) )
##
minfcn = rfit.MinFcnValue()
s , n = pretty_float ( minfcn )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Minimal FCN" , n , ' ' + s ) )
## 7.Probability in %[%]
prob = rfit.Prob() / 100
rows.append ( ( "Probability" , '[%]' , ' %5.3e' % prob ) )
## 8. distrance to minimum
edm = rfit.Edm()
s , n = pretty_float ( edm )
if n : n = '[10^%+d]' % n
else : n = ''
rows.append ( ( "Estimated distance to minimum" , n , ' ' + s ) )
ncalls = rfit.NCalls()
rows.append ( ( "FCN calls" , '' , ' ' + '%d' % ncalls ) )
##
has_minos = False
for i in rfit :
if not rfit.HasMinosError( i ) : continue
has_minos = True
break
if has_minos :
rows = [ row + ('','','') for row in rows ]
header = header + ( 'neg-minos' , 'pos-minos' , 'Global corr.' )
else :
rows = [ row + ('',) for row in rows ]
header = header + ( 'Global corr.' , )
for i in rfit :
pname = rfit.GetParameterName ( i )
value = rfit.Value ( i )
fixed = rfit.IsParameterFixed ( i )
fmte = ''
if fixed :
v = value
s , n = pretty_float ( v )
s = s + '(fixed)'
nv = n
else :
error = rfit.Error ( i )
v = VE ( value , error * error )
##
fmt , fmtv , fmte , n = fmt_pretty_ve ( v )
s = fmt % ( value / 10**n , error / 10**n )
nv = n
if n : n = '[10^%+d]' % n
else : n = ''
pname = "%-2d: %s"% ( i , pname )
row = pname , n , ' ' + s
if not fixed and rfit.HasMinosError( i ) :
if fmte :
error_low = fmte % ( rfit.LowerError ( i ) / 10**nv )
error_up = fmte % ( rfit.UpperError ( i ) / 10**nv )
else :
error_low = "%+8g" % ( rfit.LowerError ( i ) / 10**nv )
error_up = "%+8g" % ( rfit.UpperError ( i ) / 10**nv )
else :
error_low = ''
error_up = ''
if has_minos :
row = row + ( error_low , error_up )
gcc = rfit.GlobalCC ( i ) * 100
gcc = '%+5.1f%%' % gcc
row = row + ( gcc, )
rows.append ( row )
if not title : title = rfit.GetTitle()
import ostap.logger.table as T
rows = [ header ] + rows
return T.table ( rows , title = title , prefix = prefix )
from ostap.core.meta_info import root_info
from ostap.logger.colorized import allright, attention
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ : logger = getLogger( 'ostap.fitting.minuit' )
else : logger = getLogger( __name__ )
# =============================================================================
logger.debug ( 'Some useful decorations for (T)Minuit functions')
# =============================================================================
partypes = integer_types
# =============================================================================
## return codes from MINUIT commands
return_codes = {
0 : allright ( 'command executed normally' ) ,
1 : attention ( 'command is blank, ignored' ) ,
2 : attention ( 'command line unreadable, ignored' ) ,
3 : attention ( 'unknown command, ignored' ) ,
4 : attention ( 'abnormal termination (e.g., MIGRAD not converged)' ),
5 : 'command is a request to read PARAMETER definitions' ,
6 : "'SET INPUT' command" ,
7 : "'SET TITLE' command" ,
8 : "'SET COVAR' command" ,
9 : 'reserved' ,
10 : 'END command' ,
11 : 'EXIT or STOP command' ,
12 : 'RETURN command' ,
}
# =============================================================================
## get the parameter from (T)Minuit
def build_bar(self):
"""Figure new percent complete, and rebuild the bar string base on self.amount.
"""
diff = float(self.amount - self.min)
done = (diff / float(self.span)) * 100.0
percent_done = int(round(done))
# figure the proper number of 'character' make up the bar
all_full = self.width - 2
num_hashes = int(round((percent_done * all_full) / 100))
if 100 <= done and self.__end is None:
self.__end = time.time()
if self.__end is None and num_hashes == self._hashes: return False
eta = ''
leta = len(eta)
if self.__end is None and 6 < num_hashes and 1 < done:
now = time.time()
feta = int((100 - done) * (now - self.__start) / done)
h, _ = divmod(feta, 3600)
m, s = divmod(feta - h * 3600, 60)
if h: eta = 'ETA %02d:%02d:%02d ' % (h, m, s)
elif m: eta = 'ETA %02d:%02d ' % (m, s)
elif s >= 1: eta = 'ETA %02d ' % s
leta = len(eta)
elif (not self.__end is None) and 5 < num_hashes:
now = self.__end
feta = int(now - self.__start)
h, _ = divmod(feta, 3600)
m, s = divmod(feta - h * 3600, 60)
if h: eta = '%02d:%02d:%02d ' % (h, m, s)
elif m: eta = '%02d:%02d ' % (m, s)
elif s >= 1: eta = '%ds ' % s
leta = len(eta)
if self.mode == 'dynamic':
# build a progress bar with self.char (to create a dynamic bar
# where the percent string moves along with the bar progress.
if eta and leta < num_hashes:
self.bar = allright(eta + self.char * (num_hashes - leta))
else:
self.bar = allright(self.char * num_hashes)
else:
# build a progress bar with self.char and spaces (to create a
# fixed bar (the percent string doesn't move)
if eta and leta + 1 < num_hashes:
self.bar = allright(eta + self.char *
(num_hashes - leta)) + ' ' * (all_full -
num_hashes)
else:
self.bar = allright(
self.char * num_hashes) + ' ' * (all_full - num_hashes)
percent_str = str(percent_done) + "%"
self.bar = '[ ' + self.bar + ' ] ' + infostr(percent_str)
self._hashes = num_hashes
self._done = done
return True
self.output.flush()
self.silent = True
def __enter__(self):
self.show()
return self
def __exit__(self, *_):
self.end()
def __del__(self):
self.end()
# =============================================================================
_bar_ = (allright('Running ... | ') + '\r', allright('Running ... / ') + '\r',
allright('Running ... - ') + '\r', allright('Running ... \\ ') + '\r')
_lbar = len(_bar_)
_done_ = infostr('Done %-d') + '\n'
# =============================================================================
## @class RunningBar
# - RunningBar
# @code
# with RunningBar () as bar :
# for i in xrange(2000) :
# ...
# bar += 1
# @endcode
# With helper function:
def makeWeights(
dataset,
plots=[],
database="weights.db",
compare=None, ## comparison function
delta=0.001, ## delta for ``mean'' weight variation
minmax=0.05, ## delta for ``minmax'' weight variation
power=0, ## auto-determination
debug=True, ## save intermediate information in DB
tag="Reweighting"):
assert 0 < delta, "makeWeights(%s): Invalid value for ``delta'' %s" % (
tag, delta)
assert 0 < minmax, "makeWeights(%s): Invalid value for ``minmax'' %s" % (
tag, minmax)
from ostap.logger.colorized import allright, attention, infostr
from ostap.utils.basic import isatty
power = power if power >= 1 else len(plots)
nplots = len(plots)
if 1 < nplots:
import math
fudge_factor = math.sqrt(1.0 / max(2.0, nplots - 1.0))
delta = delta * fudge_factor
minmax = minmax * fudge_factor
save_to_db = []
## number of active plots for reweighting
for wplot in plots:
what = wplot.what ## variable/function to plot/compare
how = wplot.how ## weight and/or additional cuts
address = wplot.address ## address in database
hdata0 = wplot.data ## original "DATA" object
hmc0 = wplot.mc_histo ## original "MC" histogram
ww = wplot.w ## relative weight
#
# normailze the data
#
hdata = hdata0
if isinstance(hdata, ROOT.TH1): hdata = hdata.density()
# =====================================================================
## make a plot on (MC) data with the weight
# =====================================================================
dataset.project(hmc0, what, how)
st = hmc0.stat()
mnmx = st.minmax()
if iszero(mnmx[0]):
logger.warning("Reweighting: statistic goes to zero %s/``%s''" %
(st, address))
# =====================================================================
## normalize MC
# =====================================================================
hmc = hmc0.density()
# =====================================================================
## calculate the reweighting factor : a bit conservative (?)
# this is the only important line
# =====================================================================
# try to exploit finer binning if/when possible
if isinstance ( hmc , ( ROOT.TH1F , ROOT.TH1D ) ) and \
isinstance ( hdata , ( ROOT.TH1F , ROOT.TH1D ) ) and \
len ( hmc ) >= len( hdata ) :
w = (1.0 / hmc) * hdata ## NB!
## elif isinstance ( hmc , ( ROOT.TH2F , ROOT.TH2D ) ) and \
## isinstance ( hdata , ( ROOT.TH2F , ROOT.TH2D ) ) and \
## len ( hmc.GetXaxis() ) >= len( hdata.GetXaxis () ) and \
## len ( hmc.GetYaxis() ) >= len( hdata.GetYaxis () ) : w = ( 1.0 / hmc ) * hdata ## NB!
## elif isinstance ( hmc , ( ROOT.TH3F , ROOT.TH3D ) ) and \
## isinstance ( hdata , ( ROOT.TH3F , ROOT.TH3D ) ) and \
## len ( hmc.GetXaxis() ) >= len( hdata.GetXaxis () ) and \
## len ( hmc.GetYaxis() ) >= len( hdata.GetYaxis () ) and \
## len ( hmc.GetZaxis() ) >= len( hdata.GetZaxis () ) : w = ( 1.0 / hmc ) * hdata ## NB!
else:
w = hdata / hmc ## NB!
# =====================================================================
## scale & get the statistics of weights
w /= w.stat().mean().value()
cnt = w.stat()
#
mnw, mxw = cnt.minmax()
wvar = cnt.rms() / cnt.mean()
good1 = wvar.value() <= delta
good2 = abs(mxw - mnw) <= minmax
good = good1 and good2 ## small variance?
#
afunc1 = allright if good1 else attention
afunc2 = allright if good2 else attention
#
message = "%s: %24s:" % (tag, address)
message += ' ' + 'mean=%12s' % cnt.mean().toString('(%4.2f+-%4.2f)')
message += ' ' + afunc2('min/max=%-5.3f/%5.3f' %
(cnt.minmax()[0], cnt.minmax()[1]))
message += ' ' + afunc1('rms=%s[%%]' %
(wvar * 100).toString('(%4.2f+-%4.2f)'))
logger.info(message)
#
## make decision based on the variance of weights
#
mnw, mxw = cnt.minmax()
if good: ## small variance?
message = "%s: No more reweights for %s" % (tag, address)
message += ' ' + allright("min/max/rms=%+3.1f/%+3.1f/%3.1f[%%]" %
((mnw - 1) * 100,
(mxw - 1) * 100, 100 * wvar))
logger.info(message)
del w, hdata, hmc
else:
save_to_db.append((address, ww, hdata0, hmc0, hdata, hmc, w))
# =====================================================================
## make a comparison (if needed)
# =====================================================================
if compare: compare(hdata0, hmc0, address)
## for single reweighting
## if 1 == nplots : power = 1
## if power != nplots :
# logger.info ( "%s: ``power'' is %g/#%d" % ( tag , power , nplots ) )
active = [p[0] for p in save_to_db]
all = [p.address for p in plots]
for i, a in enumerate(all):
if a in active:
if isatty(): all[i] = attention(a)
else: all[i] = '*' + a + '*'
else:
if isatty(): all[i] = allright(a)
logger.info("%s: reweights are: %s" % (tag, (', '.join(all))))
## if len ( active ) != nplots :
## if database and save_to_db :
## power += ( nplots - len ( active ) )
## logger.info ("%s: ``power'' is changed to %g" % ( tag , power ) )
nactive = len(active)
while database and save_to_db:
entry = save_to_db.pop()
address, ww, hd0, hm0, hd, hm, weight = entry
## eff_exp = 1.0 / power
## eff_exp = 0.95 / ( 1.0 * nactive ) ** 0.5
cnt = weight.stat()
mnw, mxw = cnt.minmax()
if 0.95 < mnw and mxw < 1.05:
eff_exp = 0.75 if 1 < nactive else 1.50
elif 0.90 < mnw and mxw < 1.10:
eff_exp = 0.70 if 1 < nactive else 1.30
elif 0.80 < mnw and mxw < 1.20:
eff_exp = 0.65 if 1 < nactive else 1.25
elif 0.70 < mnw and mxw < 1.30:
eff_exp = 0.60 if 1 < nactive else 1.15
elif 0.50 < mnw and mxw < 1.50:
eff_exp = 0.55 if 1 < nactive else 1.10
else:
eff_exp = 0.50 if 1 < nactive else 1.0
## print 'effective exponent is:', eff_exp , address , mnw , mxw , (1.0/mnw)*mnw**eff_exp , (1.0/mxw)*mxw**eff_exp
if 1 < nactive and 1 != ww:
eff_exp *= ww
logger.info("%s: apply ``effective exponent'' of %.3f for ``%s''" %
(tag, eff_exp, address))
if 1 != eff_exp and 0 < eff_exp:
weight = weight**eff_exp
## print 'WEIGHT stat', eff_exp, weight.stat()
## hmmmm... needed ? yes!
#if 1 < power : weight = weight ** ( 1.0 / power )
## relative importance
#if 1 != ww :
# logger.info ("%s: apply ``relative importance factor'' of %.3g for ``'%s'" % ( tag , ww , address ) )
# weight = weight ** ww
with DBASE.open(database) as db:
db[address] = db.get(address, []) + [weight]
if debug:
addr = address + ':REWEIGHTING'
db[addr] = db.get(addr, []) + list(entry[2:])
del hd0, hm0, hd, hm, weight, entry
return active
## variables to be used in MC-dataset
variables = [
Variable('x', 'x-var', 0, 20),
Variable('y', 'y-var', 0, 15),
]
selector = SelectorWithVars(variables,
'0<x && x<20 && 0<y && y<20',
silence=True)
mctree.process(selector, silent=True)
mcds_ = selector.data ## dataset
# =============================================================================
## start reweighting iterations:
for iter in range(1, maxIter + 1):
tag = 'Reweighting iteration #%d' % iter
logger.info(allright(tag))
with timing(tag + ': prepare MC-dataset:', logger=logger):
# =========================================================================
## 0) The weighter object
weighter = Weight(dbname, weightings)
# =========================================================================
## 1a) create new "weighted" mcdataset
mcds = mcds_.Clone()
with timing(tag + ': add weight to MC-dataset', logger=logger):
## 1b) add "weight" variable to dataset
mcds.add_reweighting(weighter, name='weight')
if 1 == iter % 10: logger.info((tag + ' MCDATA:\n%s') % mcds)
def the_table(rows,
title='',
prefix='',
alignment=(),
wrap_width=-1,
indent=wrap_indent):
"""Format the list of rows as a table (home-made primitive)
- Each row is a sequence of column cells.
- The first row defines the column headers.
>>> table_data = [
... ( 'Name' , 'Occupation' , 'Note' ) ,
... ( 'Alice' , '?' , '---' ) ,
... ( 'Bob' , 'unemployed' , '' ) ]
>>> t = the_table ( table_data , 'Title' )
>>> print (t)
"""
## calculate the number of columns
rows = list(rows)
nc = 0
for row in rows:
nc = max(nc, len(row))
wraps = []
for i, a in zip(range(nc), alignment):
if a and isinstance(a, str):
al = a.lower()
if al in left: pass
elif al in right: pass
elif al in wrapped: wraps.append(i)
## calculate the maximum width for columns
widths = {}
for k, row in enumerate(rows):
cols = [c for c in row]
while len(cols) < nc:
cols.append('')
for i, c in enumerate(cols):
if not i in widths: widths[i] = 1
widths[i] = max(widths[i], len(decolorize(c)))
cols = tuple(cols)
rows[k] = cols
## play with wrapped columns
while wraps:
twidth = 1 + len(prefix)
for k in widths:
if not k in wraps:
twidth += widths[k] + 2
twidth += nc + 1
_, w = terminal_size()
if w <= twidth: break
nw = len(wraps)
ww = (w - twidth) - 2 * nw
ww, _ = divmod(ww, nw)
if 12 < wrap_width and wrap_width < ww:
ww = wrap_width
if ww < 15: break
lw = len(wraps)
wraps = [i for i in wraps if ww <= widths[i]]
if len(wraps) == lw: break
for i in wraps:
widths[i] = min(ww, widths[i])
hformats = ["{:^%d}" % widths[c] for c in range(nc)]
rformats = [" {:^%d} " % widths[c] for c in range(nc)]
for i, a in zip(range(nc), alignment):
if a and isinstance(a, str):
al = a.lower()
if al in left or al in wrapped:
hformats[i] = hformats[i].replace('^', '<')
rformats[i] = rformats[i].replace('^', '<')
elif al in right:
hformats[i] = hformats[i].replace('^', '>')
rformats[i] = rformats[i].replace('^', '>')
if wraps:
rows_ = rows
rows = []
for row in rows_:
cells = []
for i, c in enumerate(row):
if i in wraps and wrap_width < len(c):
cells.append(textwrap.wrap(indent + c, widths[i]))
else:
cells.append([c])
nr = 0
for c in cells:
nr = max(nr, len(c))
for l in cells:
while len(l) < nr:
l.insert(0, '')
l.append('')
for r in range(nr):
new_row = []
for i, c in enumerate(cells):
lc = len(c)
if r < lc: new_row.append(c[r])
else: new_row.append('')
rows.append(new_row)
totwidth = 0
for c in widths:
totwidth += widths[c]
totwidth += (nc - 1) * 3
if totwidth < len(title):
delta = 1 + (len(title) - totwidth) // nc
for c in widths:
widths[c] += delta
seps = ['-' * (widths[c] + 2) for c in range(nc)]
sepline = '+' + "+".join(seps) + '+'
table = []
if title:
sline = '+' + '-' * (len(sepline) - 2) + '+'
tfmt = "{:^%d}" % (len(sepline) - 4)
t = '| ' + allright(tfmt.format(decolorize(title))) + ' |'
table.append(sline)
table.append(t)
table.append(sepline)
for i, row in enumerate(rows):
cols = [c for c in row]
while len(cols) < nc:
cols.append('')
if 0 == i:
table.append('| ' + ' | '.join([
infostr(f.format(decolorize(i)))
for (f, i) in zip(hformats, cols)
]) + ' |')
table.append(sepline)
else:
table.append('|' + '|'.join(
[f.format(decolorize(i))
for (f, i) in zip(rformats, cols)]) + '|')
table.append(sepline)
return prefix + ('\n' + prefix).join(table) if prefix else '\n'.join(table)
def _mn_table_ ( self , title = '' , prefix = '' ) :
"""Build parameter table from (T)Minuit
"""
header = ( 'Parameter' , '' , 'Value' )
rows = []
from ostap.fitting.utils import fit_status , cov_qual
from ostap.logger.utils import pretty_float, pretty_ve, pretty_2ve
status = self.GetStatus()
if status :
status = fit_status ( status )
row = ' Status' , '' , status
rows.append ( row )
stat = _mn_stat_ ( self )
istat = stat.pop ( 'ISTAT' , None )
if not istat is None :
cq = ''
if -1 == istat : cq = cov_qual ( istat )
elif 3 == istat : cq = allright ( cov_qual ( istat ) )
elif istat in ( 0 , 1 , 2 ) : cq = attentiont ( cov_qual ( istat ) )
else : cq = cov_qual ( istat )
row = 'Covariance matrix quality' , '' , cq
rows.append ( row )
fmin = stat.pop ( 'FMIN' , None )
if not fmin is None :
s , n = pretty_float ( fmin )
if n : n = '[10^%+d]' % n
else : n = ''
row = 'Minimized FCN value' , n , s
rows.append ( row )
fedm = stat.pop ( 'FEDM' , None )
if not fedm is None :
s , n = pretty_float ( fedm )
if n : n = '[10^%+d]' % n
else : n = ''
row = 'Estimated distance to minimum' , n , s
rows.append ( row )
errdef = stat.pop ( 'ERRDEF' , None )
## needed ?
has_limits = False
has_minos = False
val = ctypes.c_double ( 0 )
err = ctypes.c_double ( 0 )
low = ctypes.c_double ( 0 )
high = ctypes.c_double ( 0 )
idx = ctypes.c_int ( 0 )
dct_pars = {}
## loop over all parameters
for i in self :
name = ROOT.TString()
self.mnpout ( i , name , val , err , low , high , idx )
if not 0 <= idx.value : continue
dct = {}
dct [ 'name' ] = '%-2d: %s' % ( i , str ( name ).strip() )
dct [ 'value'] = val.value
if low.value < high.value :
dct [ 'low' ] = low .value
dct [ 'high'] = high.value
has_limits = True
if 0 <= err.value :
dct [ 'error' ] = err.value
mn_plus , mn_minus = _mn_minerr_ ( self , str ( name ) )
if 0 < mn_plus or mn_minus < 0 :
dct [ 'minos+' ] = mn_plus
dct [ 'minos-' ] = mn_minus
has_minos = True
dct_pars[ i ] = dct
if has_minos :
## some parameters have MINOS errors, add columns
header = ( header ) + ( 'neg-minos' , 'pos-minos' )
rows = [ r + ( '','' ) for r in rows ]
if has_limits :
## some parameters have LIMITS, add columns
header = ( header ) + ( 'low limit' , 'high limit' )
rows = [ r + ( '','' ) for r in rows ]
for p in dct_pars :
pdict = dct_pars [ p ]
row = []
row.append ( pdict.pop ( 'name' ) )
value = pdict.pop ( 'value' )
error = pdict.pop ( 'error', None )
if error is None :
s , n = pretty_float ( value )
s = "%s(fixed)" % s
else :
s , n = pretty_ve ( VE ( value , error * error ) )
if n : row.append ( '[10^%+d]' % n )
else : row.append ( '' )
row.append ( s )
if has_minos :
mn_plus = pdict.pop ( 'minos+' , None )
mn_minus = pdict.pop ( 'minos-' , None )
if mn_plus is None : mn_plus = ''
else : mn_plus = '%8f' % ( mn_plus * 10** n )
if mn_minus is None : mn_minus = ''
else : mn_minus = '%8f' % ( mn_minus * 10** n )
row.append ( mn_minus )
row.append ( mn_plus )
if has_limits :
low = pdict.pop ( 'low' , None )
high = pdict.pop ( 'high' , None )
if low is None : low = ''
else : low = '%8f' % ( low * 10 ** n )
if high is None : high = ''
else : high = '%8f' % ( high * 10 ** n )
row.append ( low )
row.append ( high )
row = tuple ( row )
rows.append ( row )
rows = [ header ] + rows
from ostap.logger.table import table
return table ( rows , title = title , prefix = prefix )
# =============================================================================
## variables to be used in MC-dataset
variables = [
Variable('x', 'x-var', 0, 20),
Variable('y', 'y-var', 0, 15),
]
selector = SelectorWithVars(variables,
'0<x && x<20 && 0<y && y<20',
silence=True)
mctree.process(selector, silent=True)
mcds_ = selector.data ## dataset
# =============================================================================
## start reweighting iterations:
for iter in range(1, maxIter + 1):
logger.info(allright('Reweighting iteration %d ' % iter))
with timing('Prepare MC-dataset:', logger=logger):
# =========================================================================
## 0) The weighter object
weighter = Weight(dbname, weightings)
# =========================================================================
## 1a) create new "weighted" mcdataset
mcds = mcds_.Clone()
with timing('Add weight to MC-dataset', logger=logger):
## 1b) add "weight" variable to dataset
mcds.add_reweighting(weighter, name='weight')
logger.info('MCDATA:\n%s' % mcds)
def _rfr_table_(r, title='', prefix=''):
""" print RooFitResult as a table
>>> result = ...
>>> result.table()
"""
from ostap.fitting.utils import fit_status, cov_qual
rows = []
if r.status():
row = attention(' Status'), '', attention(fit_status(r.status())), ''
rows.append(row)
s, n = pretty_float(r.minNll())
if n: n = '[10^%+d]' % n
else: n = ''
rows.append(("Minimized FCN/NLL value", n, ' ' + s, ''))
s, n = pretty_float(r.edm())
if n: n = '[10^%+d]' % n
else: n = ''
rows.append(('Estimated distance to minimum', n, ' ' + s, ''))
cq = r.covQual()
cn = ''
if -1 == cq:
cn = cov_qual(cq)
elif 3 == cq:
cn = allright(cov_qual(cq))
elif cq in (0, 1, 2):
cn = attention(cov_qual(cq))
else:
cn = cov_qual(cq)
rows.append(('Covariance matrix quality', '', ' ' + cn, ''))
for i in range(r.numStatusHistory()):
label = r.statusLabelHistory(i)
code = r.statusCodeHistory(i)
row = 'Status: %s ' % label, '', '%d' % code
if not code in (0, -1):
row = attention(row[0]), row[1], ' ' + attention(row[2]), ''
else:
row = row[0], row[1], ' ' + allright(row[2]), ''
rows.append(row)
nbadnll = r.numInvalidNLL()
if 0 < nbadnll:
rows.append(('Invalid FCN/NLL evaluations', '', ' %d' % nbadnll, ''))
rows = [('', 'Unit', 'Value', 'Global/max correlation')] + rows
pars_all = r.params(float_only=False)
pars_float = r.params(float_only=True)
## constant/fix parameters
crows = []
for p in pars_all:
if p in pars_float: continue
v, a = pars_all[p]
s, n = pretty_float(v.value())
if n: n = '[10^%+d]' % n
else: n = ''
row = p, n, ' ' + s, ''
crows.append(row)
## floating parameters
frows = []
for p in pars_float:
v, a = pars_float[p]
if not a.hasAsymError():
s, n = pretty_ve(v)
else:
s, n = pretty_2ve(a.getVal(), a.getAsymErrorHi(),
a.getAsymErrorLo())
if n: n = '[10^%+d]' % n
else: n = ''
cc = 'Not available'
if 0 <= cq:
mxr, mxv = r.max_cor(p)
gc = r.globalCorr(p)
cc = '%+5.3f/(%+5.3f,%s)' % (gc, mxr, mxv)
if 0.95 < abs(gc) or 0.95 < abs(mxr): cc = attention(cc)
row = p, n, s, cc
frows.append(row)
crows.sort()
frows.sort()
all = rows + crows + frows
import ostap.logger.table as T
all = T.align_column(all, 0, 'left')
all = T.align_column(all, 1, 'left')
all = T.align_column(all, 2, 'left')
all = T.align_column(all, 3, 'left')
for l in range(len(rows), len(all)):
line = all[l]
line = list(line)
line[0] = allright(line[0])
all[l] = tuple(line)
if title:
return T.table(all, title=title, prefix=prefix)
else:
return T.table(all, title=r.GetTitle(), prefix=prefix)
