def __call__ ( self , *x ) :
n = self.__N
assert len ( x ) == n , 'Invalid argument size'
xve = [ VE(i) for i in x ] ## force everything to be VE
xv = [ i.value() for i in xve ] ## get only the values
xv = tuple ( xv )
## value of the function
value = self.__func ( *xv )
## calculate the covariance
cov2 = 0
for i in range ( n ) :
c2 = xve[i].cov2()
if c2 <= 0 or iszero ( c2 ) : continue
## calculate the gradient
df = self.partial[i] ( *xv )
if iszero ( df ) : continue
cov2 += c2 * df**2
return VE ( value , cov2 )
def __call__ ( self , args , cor = None ) :
"""Calcualte the value of the scalar function of N arguments
with uncertainties and correlations
>>> fun2 = lambda x , y : ...
>>> fun2e = EvalNVEcor ( fun2 , 2 )
>>> cor = ... # 2x2 symmetric correlation matrix
>>> x = ...
>>> y = ...
>>> result = fun2e ( (x,y) , cor ) ##
"""
n = self.N
assert len ( args ) == n , 'Invalid argument size'
## get value of the function
val = self.func ( *args )
c2 = 0
x = n * [ 0 ] ## argument
g = n * [ 0 ] ## gradient
for i in range ( n ) :
xi = VE ( args[i] )
x [i] = x
ci = xi.cov2()
if ci < 0 or iszero ( ci ) : continue
di = self.partial[i] ( *args )
if iszero ( di ) : continue
ei = xi.error()
g [i] = di
e [i] = ei
## diagonal correlation coefficients are assumed to be 1 and ignored!
c2 += ci * di * di
for j in range ( i ) :
xj = x [ j ]
cj = xj.cov2 ()
if cj < 0 or iszero ( cj ) : continue
dj = d [ j ]
if iszero ( dj ) : continue
ej = e [ j ]
rij = self.__corr ( cor , i , j ) if cor else 0
assert -1 <= rij <= 1 or isequal ( abs ( rij ) , 1 ) ,\
'Invalid correlaation coefficient (%d,%d)=%s ' % ( i , j , rij )
c2 += 2.0 * di * dj * rij * ei * ej
return VE ( val , c2 )
def fmt_pretty_float(value, width=8, precision=6):
"""Format for nice printout of the floating number
- return format for nice string and the separate exponent
>>> fmt , n = fmt_pretty_float ( number )
"""
from ostap.core.ostap_types import integer_types, num_types
assert isinstance ( value , num_types ),\
'Invalid value parameter %s/%s' % ( value , type ( value ) )
## not finite?
from ostap.math.base import isfinite
if not isfinite(value): return "%s", 0
assert isinstance ( width , integer_types ) and \
isinstance ( precision , integer_types ) and 2 <= precision < width, \
"Invalid width/precision parameters %s/%s" % ( width , precision )
v = value
av = abs(v)
if 100 <= av < 1000:
fmt2 = '%%+%d.%df' % (width, precision - 2)
return fmt2, 0
elif 10 <= av < 100:
fmt1 = '%%+%d.%df' % (width, precision - 1)
return fmt1, 0
elif 1 <= av < 10:
fmt0 = '%%+%d.%df' % (width, precision)
return fmt0, 0
elif 0.1 <= av < 1:
fmt0 = '%%+%d.%df' % (width, precision)
return fmt0, 0
from ostap.math.base import frexp10, iszero
if iszero(av):
fmt0 = '%%+%d.%df' % (width, precision)
return fmt0, 0
v_a, v_e = frexp10(v)
if 0 == v_e and iszero(v_a):
fmt0 = '%%+0%d.%df' % (width, precision)
return fmt0, 0
v_a *= 10
v_e -= 1
n, r = divmod(v_e, 3)
ra = v_a * (10**r)
fmt, p = fmt_pretty_float(ra, width, precision)
return fmt, p + 3 * n
def optimal_step(self, fun, x, h, hmax=-1, args=(), kwargs={}):
"""Get the optimal step (and f(x)) for numerical differentiation
>>> deLevie = ...
>>> hopt , f0 = deLevier.optimal_step ( fun , x = 0 , h = 0.1 )
"""
## adjust inital step size
h = self.adjust_step(x, h, hmax)
## get the function value at the given point
f0 = fun(x, *args, **kwargs)
dx = delta(x)
xph = x + h
h = xph - x
h_max = hmax if dx < hmax else self.max_step
## adjust h
if 0 < h_max < abs(h):
h = math.copysign(h_max, h)
i = self.__I
j = self.__J
## if the intial step is too small, choose another one
eps_j = self.table2[3]
if abs(h) < eps_j or abs(h) < dx:
if iszero(x): h = self.deltas[i]
else: h = (1 + abs(x)) * eps_j ## Sect. 7
## 1) find the estimate for first and "J"th the derivative with the given step
d1, dJ = self.derivatives(True, fun, x, h, args=args, kwargs=kwargs)
## find the optimal step
if iszero(dJ) or (iszero(f0) and iszero(x * d1)):
if iszero(x): hopt = self.deltas[i]
else: hopt = (1 + abs(x)) * eps_j ## Sect. 7
else:
d_j = self.table2[2]
hopt = d_j * ((abs(f0) + abs(x * d1)) / abs(dJ))**(1.0 / j)
## final adjustment
hopt = self.adjust_step(x, hopt, hmax)
return hopt, f0
def __init__(
self,
name,
x, ## the first dimension
y, ## the second dimension
n=2, ## polynomial degree in X and Y
tau=None): ## the exponent
if x.getMin() != y.getMin():
logger.warning(
'PSPos2Dsym: x&y have different low edges %s vs %s' %
(x.getMin(), y.getMin()))
if x.getMax() != y.getMax():
logger.warning(
'PSPos2Dsym: x&y have different high edges %s vs %s' %
(x.getMax(), y.getMax()))
PDF2.__init__(self, name, x, y)
#
## get tau
#
taumax = 100
mn, mx = x.minmax()
mc = 0.5 * (mn + mx)
#
if not iszero(mn): taumax = 100.0 / abs(mn)
if not iszero(mc): taumax = min(taumax, 100.0 / abs(mc))
if not iszero(mx): taumax = min(taumax, 100.0 / abs(mx))
#
## the exponential slopes
#
self.tau = makeVar(tau, "tau_%s" % name, "tau(%s)" % name, tau, 0,
-taumax, taumax)
#
self.m1 = x ## ditto
self.m2 = y ## ditto
#
num = (n + 1) * (n + 2) / 2 - 1
self.makePhis(num)
#
#
## finally build PDF
#
self.pdf = Ostap.Models.Expo2DPolSym('exp2Ds_%s' % name,
'Expo2DPolSym(%s)' % name, self.x,
self.y, self.tau, n,
self.phi_list)
def __init__(
self,
name, ## the name
mass, ## the variable
alpha=None, ## the slope of the first exponent
delta=None, ## (alpha+delta) is the slope of the first exponent
x0=0, ## f(x)=0 for x<x0
power=0, ## degree of polynomial
tau=None): ##
#
PDF.__init__(self, name)
#
self.mass = mass
self.power = power
#
mn, mx = mass.minmax()
mc = 0.5 * (mn + mx)
taumax = 100
#
if not iszero(mn): taumax = 100.0 / abs(mn)
if not iszero(mc): taumax = min(taumax, 100.0 / abs(mc))
if not iszero(mx): taumax = min(taumax, 100.0 / abs(mx))
#
## the exponential slope
#
self.alpha = makeVar(alpha, "alpha_%s" % name, "#alpha(%s)" % name,
alpha, 1, 0, taumax)
self.delta = makeVar(delta, "delta_%s" % name, "#delta(%s)" % name,
delta, 1, 0, taumax)
self.x0 = makeVar(x0, "x0_%s" % name, "x_{0}(%s)" % name, x0, mn,
mn - 0.5 * (mx - mn), mx + 0.5 * (mx - mn))
#
#
if 0 >= self.power:
self.phis = []
self.phi_list = ROOT.RooArgList()
self.pdf = ROOT.RooExponential('exp_%s' % name, 'exp(%s)' % name,
mass, self.tau)
else:
#
self.makePhis(power)
#
self.pdf = Ostap.Models.TwoExpoPositive(
'2expopos_%s' % name, '2expopos(%s)' % name, mass,
self.alpha, self.delta, self.x0, self.phi_list, mass.getMin(),
mass.getMax())
def derivative(fun, x, h=0, I=2, err=False):
"""Calculate the first derivative for the function
# @code
# >>> fun = lambda x : x*x
# >>> print derivative ( fun , x = 1 )
- see R. De Levie, ``An improved numerical approximation for the first derivative''
- see https://link.springer.com/article/10.1007/s12039-009-0111-y
- see http://www.ias.ac.in/chemsci/Pdf-Sep2009/935.pdf
"""
func = lambda x: float(fun(x))
## get the function value at the given point
f0 = func(x)
## adjust the rule
I = min(max(I, 1), 8)
J = 2 * I + 1
_dfun_ = _funcs_[I]
delta = _delta_(x)
## if the intial step is too small, choose another one
if abs(h) < _numbers_[I][3] or abs(h) < delta:
if iszero(x): h = _numbers_[0][I]
else: h = abs(x) * _numbers_[I][3]
h = max(h, 2 * delta)
## 1) find the estimate for first and "J"th the derivative with the given step
d1, dJ = _dfun_(func, x, h, True)
## find the optimal step
if iszero(dJ) or (iszero(f0) and iszero(x * d1)):
if iszero(x): hopt = _numbers_[0][I]
else: hopt = abs(x) * _numbers_[I][3]
else:
hopt = _numbers_[I][2] * ((abs(f0) + abs(x * d1)) / abs(dJ))**(1.0 / J)
## finally get the derivative
if not err: return _dfun_(func, x, hopt, False)
## estimate the uncertainty, if needed
d1, dJ = _dfun_(func, x, hopt, True)
e = _numbers_[I][1] / _numbers_[I][2] * J / (J - 1)
e2 = e * e * (J * _eps_ + abs(f0) +
abs(x * d1))**(2 - 2. / J) * abs(dJ)**(2. / J)
return VE(d1, 4 * e2)
def _ve_b2s_(s):
"""Get background-over-signal ratio B/S estimate from the equation:
error(S) = 1/sqrt(S) * sqrt ( 1 + B/S).
>>> v = ...
>>> b2s = v.b2s() ## get B/S estimate
"""
#
vv = s.value()
if vv <= 0 or iszero(vv): return VE(-1, 0)
#
c2 = s.cov2()
if c2 <= 0 or iszero(c2): return VE(-1, 0)
elif isequal(vv, c2): return VE(1, 0)
elif c2 < vv: return VE(-1, 0)
#
return c2 / s - 1.0
def _solve_(func, fval, xmn, xmx, *args):
##
if isequal(xmn, xmx): return xmn
##
ifun = lambda x, *a: func(x, *a) - fval
##
fmn = ifun(xmn)
if iszero(ifun(xmn)): return xmn
fmx = ifun(xmx)
if iszero(ifun(xmx)): return xmx
##
if 0 < fmx * fmn: ## more or less arbitrary choice
return xmx if abs(fmx) <= abs(fmn) else xmn
#
from ostap.math.rootfinder import findroot
return findroot(ifun, xmn, xmx, args=args)
def __call__(self, x, *args):
"""Evaluate the function taking into account uncertainty in the argument
>>> import math
>>> x = VE(1,0.1**2)
>>> sin1 = EvalVE( math.sin , lambda s : math.cos(s) )
>>> print 'sin1(x) = %s ' % sin1(x)
>>> sin2 = EvalVE( math.sin )
>>> print 'sin2(x) = %s ' % sin2(x)
"""
#
## evaluate the function
val = self._value_(x, *args)
#
## no uncertainties?
if isinstance(x, (float, int, long)):
return VE(val, 0)
# ignore small or invalid uncertanties
elif 0 >= x.cov2() or iszero(x.cov2()):
return VE(val, 0)
# evaluate the derivative
d = self._deriv(float(x), *args)
## calculate the variance
cov2 = d * d * x.cov2()
## get a final result
return VE(val, cov2)
def __init__(
self,
name, ## the name
mass, ## the variable
power=0, ## degree of polynomial
tau=None, ## exponential slope
the_phis=None): ## the phis...
#
PDF.__init__(self, name)
#
self.mass = mass
self.power = power
#
mn, mx = mass.minmax()
mc = 0.5 * (mn + mx)
taumax = 100
#
if not iszero(mn): taumax = 100.0 / abs(mn)
if not iszero(mc): taumax = min(taumax, 100.0 / abs(mc))
if not iszero(mx): taumax = min(taumax, 100.0 / abs(mx))
#
## the exponential slope
#
self.tau = makeVar(tau, "tau_%s" % name, "tau(%s)" % name, tau, 0,
-taumax, taumax)
#
if the_phis:
## copy phis
self.makePhis(power, the_phis=the_phis) ## copy phis
elif 0 >= self.power:
self.phis = []
self.phi_list = ROOT.RooArgList()
self.pdf = ROOT.RooExponential('exp_%s' % name, 'exp(%s)' % name,
mass, self.tau)
else:
#
self.makePhis(power)
#
self.pdf = Ostap.Models.ExpoPositive('expopos_%s' % name,
'expopos(%s)' % name, mass,
self.tau, self.phi_list,
mass.getMin(), mass.getMax())
def _solve_ ( func , fval , xmn , xmx , *args ) :
##
if isequal ( xmn , xmx ) : return xmn
##
ifun = lambda x,*a : func(x,*a) - fval
##
fmn = ifun ( xmn )
if iszero ( ifun ( xmn ) ) : return xmn
fmx = ifun ( xmx )
if iszero ( ifun ( xmx ) ) : return xmx
##
if 0 < fmx * fmn : ## more or less arbitrary choice
return xmx if abs ( fmx ) <= abs ( fmn ) else xmn
#
## use scipy to find solution
from scipy import optimize
return optimize.brentq ( ifun , xmn , xmx , args = args )
def derivative(fun, x, h=0, I=2, err=False):
"""Calculate the first derivative for the function
# @code
# >>> fun = lambda x : x*x
# >>> print derivative ( fun , x = 1 )
"""
func = lambda x: float(fun(x))
## get the function value at the given point
f0 = func(x)
## adjust the rule
I = min(max(I, 1), 8)
J = 2 * I + 1
_dfun_ = _funcs_[I]
delta = _delta_(x)
## if the intial step is too small, choose another one
if abs(h) < _numbers_[I][3] or abs(h) < delta:
if iszero(x): h = _numbers_[0][I]
else: h = abs(x) * _numbers_[I][3]
h = max(h, 2 * delta)
## 1) find the estimate for first and "J"th the derivative with the given step
d1, dJ = _dfun_(func, x, h, True)
## find the optimal step
if iszero(dJ) or (iszero(f0) and iszero(x * d1)):
if iszero(x): hopt = _numbers_[0][I]
else: hopt = abs(x) * _numbers_[I][3]
else:
hopt = _numbers_[I][2] * ((abs(f0) + abs(x * d1)) / abs(dJ))**(1.0 / J)
## finally get the derivative
if not err: return _dfun_(func, x, hopt, False)
## estimate the uncertainty, if needed
d1, dJ = _dfun_(func, x, hopt, True)
e = _numbers_[I][1] / _numbers_[I][2] * J / (J - 1)
e2 = e * e * (J * _eps_ + abs(f0) +
abs(x * d1))**(2 - 2. / J) * abs(dJ)**(2. / J)
return VE(d1, 4 * e2)
def __call__ ( self , x , y , cxy = 0 ) :
"""Evaluate the function
>>> func2 = lambda x,y : x*x + y*y
>>> eval2 = Eval2VE ( func2 )
>>> x = VE(1,0.1**2)
>>> y = VE(2,0.1**2)
>>> print eval2(x,y) ## treat x,y as uncorrelated
>>> print eval2(x,y, 0) ## ditto
>>> print eval2(x,y,+1) ## treat x,y as 100% correlated
>>> print eval2(x,y,-1) ## treat x,y as 100% anti-correlated
"""
assert isinstance ( cxy , num_types ) and \
( abs ( cxy ) <= 1 or isequal ( abs ( cxy ) , 1 ) ) , \
'Invalid correlation coefficient %s' % cxy
x = VE ( x )
y = VE ( y )
xv = x.value()
yv = y.value()
val = self.func ( xv , yv )
xc2 = x.cov2()
yc2 = x.cov2()
x_plain = xc2 <= 0 or iszero ( xc2 )
y_plain = yc2 <= 0 or iszero ( yc2 )
#
if x_plain and y_plain : return VE ( val , 0 )
#
## here we need to calculate the uncertainties
#
dx = self.partial[0] ( xv , yv ) if not x_plain else 0
dy = self.partial[1] ( xv , yv ) if not y_plain else 0
#
cov2 = dx * dx * xc2 + dy * dy * yc2
if cxy and xc2 and yc2 :
cov2 += 2 * cxy * dx * dy * math.sqrt ( xc2 * yc2 )
return VE ( val , cov2 )
def iifun(f):
if iszero(f): x1, x2 = xmin, xmax
elif isequal(f, fm): return -yval
else:
x1 = _solve_(func, f, xmin, xmode)
x2 = _solve_(func, f, xmode, xmax)
return _integral_(func, x1, x2) - yval
def __init__(
self,
name,
x, ## the first dimension
y, ## the second dimension
psy=None, ## phase space in Y, Ostap::Math::PhaseSpaceNL
nx=2, ## polynomial degree in X
ny=2, ## polynomial degree in Y
tau=None): ## the exponent
PDF2.__init__(self, name, x, y)
#
## get tau
taumax = 100
mn, mx = x.minmax()
mc = 0.5 * (mn + mx)
#
if not iszero(mn): taumax = 100.0 / abs(mn)
if not iszero(mc): taumax = min(taumax, 100.0 / abs(mc))
if not iszero(mx): taumax = min(taumax, 100.0 / abs(mx))
#
## the exponential slope
#
self.tau = makeVar(tau, "tau_%s" % name, "tau(%s)" % name, tau, 0,
-taumax, taumax)
#
self.psy = psy
#
num = (nx + 1) * (ny + 1) - 1
self.makePhis(num)
#
#
## finally build PDF
#
self.pdf = Ostap.Models.ExpoPS2DPol('ps2D_%s' % name,
'PS2DPol(%s)' % name, self.x,
self.y, self.tau, self.psy, nx, ny,
self.phi_list)
def _laguerre_ ( x , f , d1 = None , d2 = None ) :
""" Root finding for Bernstein polnomials using Laguerre's method
- https://en.wikipedia.org/wiki/Laguerre%27s_method
"""
fn = f.norm() / 10
n = f.degree()
xmin = f.xmin ()
xmax = f.xmax ()
l = xmax - xmin
if not d1 : d1 = f.derivative()
if not d2 : d2 = d1.derivative()
if not f.xmin() <= x <= f.xmax() :
x = 0.5 * ( xmin + xmax )
if 1 == f.degree() :
p0 = f [ 0 ]
p1 = f [ 1 ]
s0 = signum ( p0 )
s1 = signum ( p1 )
return ( p1 * xmin - p0 * xmax ) / ( p1 - p0) ,
## the convergency is cubic, therefore 16 is *very* larger number of iterations
for i in range(16) :
if not xmin <= x <= xmax :
## something goes wrong: multiple root? go to derivative
break
vx = f.evaluate(x)
if iszero ( vx ) or isequal ( fn + vx , fn ) : return x,
G = d1 ( x ) / vx
H = G * G - d2 ( x ) / vx
d = math.sqrt ( ( n - 1 ) * ( n * H - G * G ) )
if G < 0 : d = -d
a = n / ( G + d )
x -= a
if isequal ( a + x , x ) : return x,
## look for derivative
r = _laguerre_ ( x , d1 , d2 , d2.derivative() )
if r : r = r[:1] + r
return r
def __call__ ( self , args , cov2 = None ) :
"""Calcualte the value of the scalar function of N (scalar) arguments
with the given NxN covariance matrix
>>> fun2 = lambda x , y : ...
>>> fun2e = EvalNVEcov ( fun2 , 2 )
>>> cov2 = ... # 2x2 symmetric covariance matrix
>>> result = fun2e ( (1,5) , cov2 ) ##
"""
n = self.N
assert len ( args ) == n , 'Invalid argument size'
## get value of the function
val = float ( self.func ( *args ) )
## no covariance matrix is specified ?
if not cov2 : return val
c2 = 0
g = n * [ 0 ] ## gradient
for i in range ( n ) :
di = self.partial[i] ( *args )
if iszero ( di ) : continue
g [i] = di
cii = self.__cov2 ( cov2 , i , i )
c2 += di * di * cii
for j in range ( i ) :
dj = g [ j ]
if iszero ( dj ) : continue
cij = self.__cov2 ( cov2 , i , j )
c2 += 2 * di * dj * cij
return VE ( val , c2 )
def _ve_purity_(s):
"""Calculate the ``effective purity'' ratio using the identity
p = S/(S+B) = 1/( 1 + B/S ),
- and the effective ``background-to-signal'' ratio B/S is estimated as
B/S = sigma^2(S)/S - 1
- Finally one gets
p = S / sigma^2(S)
- see Ostap::Math::b2s
- see Ostap::Math::purity
"""
#
vv = s.value()
if vv <= 0 or iszero(vv): return VE(-1, 0)
#
c2 = s.cov2()
if c2 <= 0 or iszero(c2): return VE(-1, 0)
elif isequal(vv, c2): return VE(1, 0)
elif c2 < vv: return VE(-1, 0)
#
return s / cov2
def __call__(self, x, y, cxy=0):
"""Evaluate the function
>>> func2 = lambda x,y : x*x + y*y
>>> eval2 = Eval2VE ( func2 )
>>> x = VE(1,0.1**2)
>>> y = VE(2,0.1**2)
>>> print eval2(x,y) ## treat x,y as uncorrelated
>>> print eval2(x,y, 0) ## ditto
>>> print eval2(x,y,+1) ## treat x,y as 100% correlated
>>> print eval2(x,y,-1) ## treat x,y as 100% anti-correlated
"""
## evaluate the function
val = self._value_(x, y)
#
x = VE(x)
y = VE(y)
#
x_plain = x.cov2() <= 0 or iszero(x.cov2())
y_plain = y.cov2() <= 0 or iszero(y.cov2())
#
if x_plain and y_plain: return VE(val, 0)
#
## here we need to calculate the uncertainties
#
cov2 = 0.0
#
fx = self.dFdX(x, y.value()) if not x_plain else 0
fy = self.dFdY(x.value(), y) if not y_plain else 0
#
if not x_plain: cov2 += fx * fx * x.cov2()
if not y_plain: cov2 += fy * fy * y.cov2()
#
if not x_plain and not y_plain:
## adjust the correlation coefficient:
cxy = min(max(-1, cxy), 1)
if not iszero(cxy):
cov2 += 2 * cxy * fx * fy * x.error() * y.error()
return VE(val, cov2)
def __init__(
self,
name,
x, ## the first dimension
y, ## the second dimension
nx=2, ## polynomial degree in X
ny=2, ## polynomial degree in Y
taux=None, ## the exponent in X
tauy=None): ## the exponent in Y
PDF2.__init__(self, name, x, y)
#
## get tau_x
#
xtaumax = 100
mn, mx = x.minmax()
mc = 0.5 * (mn + mx)
#
if not iszero(mn): xtaumax = 100.0 / abs(mn)
if not iszero(mc): xtaumax = min(xtaumax, 100.0 / abs(mc))
if not iszero(mx): xtaumax = min(xtaumax, 100.0 / abs(mx))
ytaumax = 100
mn, mx = y.minmax()
mc = 0.5 * (mn + mx)
#
if not iszero(mn): ytaumax = 100.0 / abs(mn)
if not iszero(mc): ytaumax = min(ytaumax, 100.0 / abs(mc))
if not iszero(mx): ytaumax = min(ytaumax, 100.0 / abs(mx))
#
## the exponential slopes
#
self.taux = makeVar(taux, "taux_%s" % name, "taux(%s)" % name, taux, 0,
-xtaumax, xtaumax)
#
self.tauy = makeVar(tauy, "tauy_%s" % name, "tauy(%s)" % name, tauy, 0,
-ytaumax, ytaumax)
#
self.m1 = x ## ditto
self.m2 = y ## ditto
#
num = (nx + 1) * (ny + 1) - 1
self.makePhis(num)
#
#
## finally build PDF
#
self.pdf = Ostap.Models.Expo2DPol('exp2D_%s' % name,
'Expo2DPol(%s)' % name, self.x,
self.y, self.taux, self.tauy, nx, ny,
self.phi_list)
def _ve_gauss_(s, accept=lambda a: True, nmax=1000):
""" Get the gaussian random number
>>> v = ... ## the number with error
## get 100 random numbers
>>> for i in range ( 0, 100 ) : print v.gauss()
## get only non-negative numbers
>>> for j in range ( 0, 100 ) : print v.gauss( lambda s : s > 0 )
"""
#
if 0 >= s.cov2() or iszero(s.cov2()): return s.value() ## return
#
v = s.value()
e = s.error()
#
for i in xrange(nmax):
r = _gauss(v, e)
if accept(r): return r
logger.warning("Can'n generate proper random number %s" % s)
return v
def __call__ ( self , s ) :
""" Calculate the weigth for the given ``event'' (==record in TTree/TChain or RooDataSet):
>>> weight = Weight ( ... )
>>> tree = ...
>>> w = weight ( tree )
"""
## initialize the weight
weight = VE(1,0)
## loop over functions
for i in self.__vars :
funval = i[1] ## accessor
functions = i[2] ## the functions
## get the weight arguments for given event
v = funval ( s )
ww = VE(1.0)
for f in functions :
if isinstance ( v , tuple ) : w = f ( *v )
else : w = f ( v )
ww *= w # update the weight factor
## keep the statistics
cnt = i[3]
cnt += ww.value()
## update the global weight
weight *= ww
vw = weight.value()
self.__counter += vw
if iszero ( vw ) : self.__nzeroes += 1
return vw
def _ms_corr_(self):
"""Get the correlation matrix
>>> mtrx = ...
>>> corr = mtrx.correlations()
"""
from math import sqrt
_t = type(self)
_c = _t()
_rows = self.kRows
_ok1 = True
_ok2 = True
for i in range(0, _rows):
ii = self(i, i)
if 0 > ii or iszero(ii):
_ok1 = False
_nan = float('nan')
for j in range(i, _rows):
_c[i, j] = _nan
continue
sii = sqrt(ii)
_c[i, i] = 1
for j in range(i + 1, _rows):
jj = self(j, j)
sjj = sqrt(jj)
ij = self(i, j)
eij = ij / (sii * sjj)
if 1 < abs(eij): _ok2 = False
_c[i, j] = eij
if not _ok1:
logger.error("correlations: zero or negative diagonal element")
if not _ok2: logger.error("correlations: invalid non-diagonal element")
return _c
def makeWeights(
dataset,
plots=[],
database="weights.db",
compare=None, ## comparison function
delta=0.001, ## delta for ``mean'' weight variation
minmax=0.05, ## delta for ``minmax'' weight variation
power=0, ## auto-determination
debug=True, ## save intermediate information in DB
tag="Reweighting"):
assert 0 < delta, "makeWeights(%s): Invalid value for ``delta'' %s" % (
tag, delta)
assert 0 < minmax, "makeWeights(%s): Invalid value for ``minmax'' %s" % (
tag, minmax)
from ostap.logger.colorized import allright, attention, infostr
from ostap.utils.basic import isatty
power = power if power >= 1 else len(plots)
nplots = len(plots)
if 1 < nplots:
import math
fudge_factor = math.sqrt(1.0 / max(2.0, nplots - 1.0))
delta = delta * fudge_factor
minmax = minmax * fudge_factor
save_to_db = []
## number of active plots for reweighting
for wplot in plots:
what = wplot.what ## variable/function to plot/compare
how = wplot.how ## weight and/or additional cuts
address = wplot.address ## address in database
hdata0 = wplot.data ## original "DATA" object
hmc0 = wplot.mc_histo ## original "MC" histogram
ww = wplot.w ## relative weight
#
# normailze the data
#
hdata = hdata0
if isinstance(hdata, ROOT.TH1): hdata = hdata.density()
# =====================================================================
## make a plot on (MC) data with the weight
# =====================================================================
dataset.project(hmc0, what, how)
st = hmc0.stat()
mnmx = st.minmax()
if iszero(mnmx[0]):
logger.warning("Reweighting: statistic goes to zero %s/``%s''" %
(st, address))
# =====================================================================
## normalize MC
# =====================================================================
hmc = hmc0.density()
# =====================================================================
## calculate the reweighting factor : a bit conservative (?)
# this is the only important line
# =====================================================================
# try to exploit finer binning if/when possible
if isinstance ( hmc , ( ROOT.TH1F , ROOT.TH1D ) ) and \
isinstance ( hdata , ( ROOT.TH1F , ROOT.TH1D ) ) and \
len ( hmc ) >= len( hdata ) :
w = (1.0 / hmc) * hdata ## NB!
## elif isinstance ( hmc , ( ROOT.TH2F , ROOT.TH2D ) ) and \
## isinstance ( hdata , ( ROOT.TH2F , ROOT.TH2D ) ) and \
## len ( hmc.GetXaxis() ) >= len( hdata.GetXaxis () ) and \
## len ( hmc.GetYaxis() ) >= len( hdata.GetYaxis () ) : w = ( 1.0 / hmc ) * hdata ## NB!
## elif isinstance ( hmc , ( ROOT.TH3F , ROOT.TH3D ) ) and \
## isinstance ( hdata , ( ROOT.TH3F , ROOT.TH3D ) ) and \
## len ( hmc.GetXaxis() ) >= len( hdata.GetXaxis () ) and \
## len ( hmc.GetYaxis() ) >= len( hdata.GetYaxis () ) and \
## len ( hmc.GetZaxis() ) >= len( hdata.GetZaxis () ) : w = ( 1.0 / hmc ) * hdata ## NB!
else:
w = hdata / hmc ## NB!
# =====================================================================
## scale & get the statistics of weights
w /= w.stat().mean().value()
cnt = w.stat()
#
mnw, mxw = cnt.minmax()
wvar = cnt.rms() / cnt.mean()
good1 = wvar.value() <= delta
good2 = abs(mxw - mnw) <= minmax
good = good1 and good2 ## small variance?
#
afunc1 = allright if good1 else attention
afunc2 = allright if good2 else attention
#
message = "%s: %24s:" % (tag, address)
message += ' ' + 'mean=%12s' % cnt.mean().toString('(%4.2f+-%4.2f)')
message += ' ' + afunc2('min/max=%-5.3f/%5.3f' %
(cnt.minmax()[0], cnt.minmax()[1]))
message += ' ' + afunc1('rms=%s[%%]' %
(wvar * 100).toString('(%4.2f+-%4.2f)'))
logger.info(message)
#
## make decision based on the variance of weights
#
mnw, mxw = cnt.minmax()
if good: ## small variance?
message = "%s: No more reweights for %s" % (tag, address)
message += ' ' + allright("min/max/rms=%+3.1f/%+3.1f/%3.1f[%%]" %
((mnw - 1) * 100,
(mxw - 1) * 100, 100 * wvar))
logger.info(message)
del w, hdata, hmc
else:
save_to_db.append((address, ww, hdata0, hmc0, hdata, hmc, w))
# =====================================================================
## make a comparison (if needed)
# =====================================================================
if compare: compare(hdata0, hmc0, address)
## for single reweighting
## if 1 == nplots : power = 1
## if power != nplots :
# logger.info ( "%s: ``power'' is %g/#%d" % ( tag , power , nplots ) )
active = [p[0] for p in save_to_db]
all = [p.address for p in plots]
for i, a in enumerate(all):
if a in active:
if isatty(): all[i] = attention(a)
else: all[i] = '*' + a + '*'
else:
if isatty(): all[i] = allright(a)
logger.info("%s: reweights are: %s" % (tag, (', '.join(all))))
## if len ( active ) != nplots :
## if database and save_to_db :
## power += ( nplots - len ( active ) )
## logger.info ("%s: ``power'' is changed to %g" % ( tag , power ) )
nactive = len(active)
while database and save_to_db:
entry = save_to_db.pop()
address, ww, hd0, hm0, hd, hm, weight = entry
## eff_exp = 1.0 / power
## eff_exp = 0.95 / ( 1.0 * nactive ) ** 0.5
cnt = weight.stat()
mnw, mxw = cnt.minmax()
if 0.95 < mnw and mxw < 1.05:
eff_exp = 0.75 if 1 < nactive else 1.50
elif 0.90 < mnw and mxw < 1.10:
eff_exp = 0.70 if 1 < nactive else 1.30
elif 0.80 < mnw and mxw < 1.20:
eff_exp = 0.65 if 1 < nactive else 1.25
elif 0.70 < mnw and mxw < 1.30:
eff_exp = 0.60 if 1 < nactive else 1.15
elif 0.50 < mnw and mxw < 1.50:
eff_exp = 0.55 if 1 < nactive else 1.10
else:
eff_exp = 0.50 if 1 < nactive else 1.0
## print 'effective exponent is:', eff_exp , address , mnw , mxw , (1.0/mnw)*mnw**eff_exp , (1.0/mxw)*mxw**eff_exp
if 1 < nactive and 1 != ww:
eff_exp *= ww
logger.info("%s: apply ``effective exponent'' of %.3f for ``%s''" %
(tag, eff_exp, address))
if 1 != eff_exp and 0 < eff_exp:
weight = weight**eff_exp
## print 'WEIGHT stat', eff_exp, weight.stat()
## hmmmm... needed ? yes!
#if 1 < power : weight = weight ** ( 1.0 / power )
## relative importance
#if 1 != ww :
# logger.info ("%s: apply ``relative importance factor'' of %.3g for ``'%s'" % ( tag , ww , address ) )
# weight = weight ** ww
with DBASE.open(database) as db:
db[address] = db.get(address, []) + [weight]
if debug:
addr = address + ':REWEIGHTING'
db[addr] = db.get(addr, []) + list(entry[2:])
del hd0, hm0, hd, hm, weight, entry
return active
def solve ( bp , C = 0 , split = 2 ) :
"""Solve equation B(x) = C, where B(x) is Bernstein polynomial, and
C is e.g. constant or another polynomial
>>> bernstein = ...
>>> roots = bernstein.solve ()
>>> roots = solve ( bernstein , C = 0 ) ## ditto
"""
## construct the equation b'(x)=0:
bp = bp.bernstein()
if C : bp = bp - C
## 1) zero-degree polynomial
if 0 == bp.degree() :
if iszero ( bp[0] ) : return bp.xmin(),
return ()
## 2) linear polynomial
elif 1 == bp.degree() :
x0 = bp.xmin()
x1 = bp.xmax()
p0 = bp[0]
p1 = bp[1]
s0 = signum ( p0 )
s1 = signum ( p1 )
bn = bp.norm()
if iszero ( p0 ) or isequal ( p0 + bn , bn ) : s0 = 0
if iszero ( p1 ) or isequal ( p1 + bn , bn ) : s1 = 0
if s0 == 0 : return x0, ##
elif s1 == 0 : return x1, ##
elif s0 * s1 > 0 : return () ## no roots
#
return ( p1 * x0 - p0 * x1 ) / ( p1 - p0) ,
## make a copy & scale is needed
bp = _scale_ ( bp + 0 )
## norm of polynomial
bn = bp.norm()
## check number of roots
nc = bp.sign_changes()
if not nc : return () ## no roots ! RETURN
## treat separetely roots at the left and right edges
roots = []
while 1 <= bp.degree() and isequal ( bp[0] + bn , bn ) :
bp -= bp[0]
bp = bp.deflate_left()
bp = _scale_ ( bp )
bn = bp.norm ()
roots.append ( bp.xmin() )
if roots : return tuple(roots) + bp.solve ( split = split )
roots = []
while 1 <= bp.degree() and isequal ( bp[-1] + bn , bn ) :
bp -= bp[-1]
bp = bp.deflate_right()
bp = _scale_ ( bp )
bn = bp.norm ()
roots.append ( bp.xmax() )
if roots : return bp.solve ( split = split ) + tuple(roots)
## check again number of roots
nc = bp.sign_changes()
if not nc : return () ## no roots ! RETURN
# =========================================================================
## there are many roots in the interval
# =========================================================================
if 1 < nc :
xmin = bp.xmin ()
xmax = bp.xmax ()
lcp = bp.left_line_hull()
if ( not lcp is None ) and xmin <= lcp <= xmax : xmin = max ( xmin , lcp )
rcp = bp.right_line_hull()
if ( not rcp is None ) and xmin <= rcp <= xmax : xmax = min ( xmax , rcp )
# =====================================================================
## Two strategies for isolation of roots:
# - use control polygon
# - use the derivative
# For both cases, the zeros of contol-polygon and/or derivatives
# are tested to be the roots of polynomial..
# If they corresponds to the roots, polinomial is properly deflated and
# deflated roots are collected
# Remaining points are used to (recursively) split interval into smaller
# intervals with presumably smaller number of roots
#
if 0 < split :
cps = bp.crossing_points()
splits = [ xmin ]
for xp in cps :
if xmin < xp < xmax : splits.append ( xp )
splits.append ( xmax )
split -= 1
else :
## use the roots of derivative
dd = bp.derivative()
rd = dd.solve ( split = split )
if not rd :
## use bisection
nrd = xmin, 0.5 * ( xmin + xmax ), xmax
else :
## use roots of derivative
nrd = [ xmin , xmax ]
for r in rd :
if xmin < r < xmax :
found = False
for rr in nrd :
if isequal ( r , rr ) :
found = True
break
if not found : nrd.append ( r )
nrd.sort()
splits = list(nrd)
## use simple bisection
if 2 >= len ( splits ) :
if xmin < xmax : splits = xmin , 0.5*( xmin + xmax ) , xmax
else : splits = bp.xmin() , 0.5*(bp.xmin()+bp.xmax()) , bp.xmax()
splits = list(splits)
roots = []
for s in splits :
bv = bp.evaluate ( s )
bn = bp.norm()
while 1 <= bp.degree() and isequal ( bv + bn , bn ) :
bp -= bv
bp = bp.deflate ( s )
bp = _scale_ ( bp )
bn = bp.norm ( )
bv = bp.evaluate ( s )
roots.append ( s )
for q in splits :
if isequal ( q , s ) :
splits.remove ( q )
if roots :
roots += list ( bp.solve ( split = ( split - 1 ) ) )
roots.sort()
return tuple(roots)
if 2 == len(splits) :
if isequal ( bp.xmin() , splits[0] ) and isequal ( bp.xmax() , splits[1] ) :
xmn = splits[0]
xmx = splits[1]
splits = [ xmn , 0.5*(xmn+xmx) , xmx ]
ns = len(splits)
for i in range(1,ns) :
xl = splits[i-1]
xr = splits[i ]
bb = _scale_ ( Bernstein ( bp , xl , xr ) )
roots += bb.solve ( split = ( split - 1 ) )
roots.sort()
return tuple ( roots ) ## RETURN
# =========================================================================
## there is exactly 1 root here
# =========================================================================
l0 = ( bp.xmax() - bp.xmin() ) / ( bp.degree() + 1 )
## trivial case
if 1 == bp.degree() :
y0 = bp[ 0]
y1 = bp[-1]
x = ( y1 * bp.xmin() - y0 * bp.xmax() ) / ( y1 - y0)
return x,
## make several iterations (not to much) for better isolation of root
for i in range ( bp.degree() + 1 ) :
xmin = bp.xmin ()
xmax = bp.xmax ()
bn = bp.norm ()
lcp = bp.left_line_hull ()
if not lcp is None :
if isequal ( lcp , xmin ) : return lcp, ## RETURN
elif lcp <= xmax or isequal ( lcp , xmax ) :
bv = bp.evaluate ( lcp )
if iszero ( bv ) or isequal ( bv + bn , bn ) : return lcp, ## RETURN
xmin = max ( xmin , lcp )
rcp = bp.right_line_hull ()
if not rcp is None :
if isequal ( lcp , xmax ) : return rcp, ## RETURN
elif lcp >= xmin or isequal ( lcp , xmin ) :
bv = bp.evaluate ( rcp )
if iszero ( bv ) or isequal ( bv + bn , bn ) : return rcp, ## RETURN
xmax = min ( xmax , rcp )
##
if isequal ( xmin , xmax ) : return 0.5*(xmin+xmax), ## RETURN
if xmin >= xmax : break ## BREAK
## avoid too iterations - it decreased the precision
if 10 * ( xmax - xmin ) < l0 : break ## BREAK
s0 = signum ( bp[ 0] )
s1 = signum ( bp[-1] )
smn = signum ( bp.evaluate ( xmin ) )
smx = signum ( bp.evaluate ( xmax ) )
## we have lost the root ?
if ( s0 * s1 ) * ( smn * smx ) < 0 : break ## BREAK
## refine the polynomial:
_bp = Bernstein ( bp , xmin , xmax )
_bp = _scale_ ( _bp )
_nc = _bp.sign_changes()
## we have lost the root
if not _nc : break ## BREAK
bp = _bp
bn = bp.norm()
# =========================================================================
## start the of root-polishing machinery
# =========================================================================
f = bp
d1 = f .derivative ()
d2 = d1.derivative ()
cps = bp.crossing_points()
if cps : x = cps[0]
else : x = 0.5*(bp.xmin()+bp.xmax())
## use Laguerre's method to refine the isolated root
l = _laguerre_ ( x , f , d1 , d2 )
return l
def solve(bs, C=0, split=5):
"""Solve equation B(x) = C
bs : (input) b-spline
C : (input) right-hand side
:Example:
>>> bs = ...
>>> print bs.solve()
"""
_bs = bs - C
if 1 >= _bs.degree():
return _bs.crossing_points(True)
## scale if needed
_bs = _scale_(_bs)
## check zeroes of control polygon
cps = _bs.crossing_points()
ncp = len(cps)
bsn = _bs.norm()
##
xmin = _bs.xmin()
xmax = _bs.xmax()
for i in range(20):
if not cps: return () ## RETURN
inserted = False
for x in cps:
if isequal(x, _bs.xmin()): continue ## CONTINUE
if isequal(x, _bs.xmax()): continue ## CONTINUE
bx = _bs(x)
if iszero(bx) or isequal(bx + bsn, bsn): continue
if _bs.insert(x): inserted = True
## all zeroes are already inserted....
if not inserted:
return cps ## RETURN
cps = _bs.crossing_points(False)
if not cps:
## roots have disapperead.. it could happen near multiple roots
cps = _bs.crossing_points(True)
## merge duplicates (if any)
cps = _merge_(cps, max(abs(xmin), abs(xmax)))
## if no convergency....
## if not cps or 1 == len(cps) or split <= 0 : return cps
if not cps or split <= 0: return cps
## come back to the original spline
_bs = bs - C
## add the current/best estimates of roots
for x in cps:
_bs.insert(x)
## get internal roots only (no edges)
_cps = cps
_bsn = _bs.norm()
left_root = False
right_root = False
if isequal(_bs[0] + _bsn, _bsn):
left_root = True
_cps = _cps[1:]
if isequal(_bs[-1] + _bsn, _bsn):
right_root = True
_cps = _cps[:-1]
cpmin = _cps[0]
cpmax = _cps[-1]
dc = cpmax - cpmin
xmin = _bs.xmin()
xmax = _bs.xmax()
dx = xmax - xmin
##
if 1 == len(cps):
x1 = 0.05 * xmin + 0.95 * cpmin
x2 = 0.05 * xmax + 0.95 * cpmax
b1 = _scale_(Ostap.Math.BSpline(_bs, xmin, x1))
b2 = _scale_(Ostap.Math.BSpline(_bs, x1, x2))
b3 = _scale_(Ostap.Math.BSpline(_bs, x2, xmax))
split -= 1
return _merge_(
solve(b1, C=0, split=split) + solve(b2, C=0, split=split) +
solve(b3, C=0, split=split), max(abs(xmin), abs(xmax)))
dl = 0.05 * min(cpmin - xmin, dc)
dr = 0.05 * min(xmax - cpmax, dc)
x_left = cpmin - dl
x_right = cpmax + dr
x_mid = 0.5 * (cpmin + cpmax)
import random
ntry = 50
for i in range(ntry):
if not isequal(_bs(x_left) + _bsn, _bsn): break
alpha = random.uniform(0.05, 1)
x_left = max(xmin, cpmin - alpha * dl)
for i in range(ntry):
if not isequal(_bs(x_mid) + _bsn, _bsn): break
alpha = random.uniform(-1, 1)
x_mid = 0.5 * (cpmin + cpmax) + alpha * dc / 20
for i in range(ntry):
if not isequal(_bs(x_right) + _bsn, _bsn): break
alpha = random.uniform(0.05, 1)
x_right = min(xmax, cpmax + alpha * dr)
## split!
split -= 1
if dc > 0.1 * dx: ## split
bs1 = _scale_(Ostap.Math.BSpline(_bs, x_left, x_mid))
bs2 = _scale_(Ostap.Math.BSpline(_bs, x_mid, x_right))
roots = _merge_(
solve(bs1, C=0, split=split) + solve(bs2, C=0, split=split),
max(abs(xmin), abs(xmax)))
else: ## trim
bst = _scale_(Ostap.Math.BSpline(_bs, x_left, x_right))
roots = _merge_(solve(bst, C=0, split=split),
max(abs(xmin), abs(xmax)))
if left_root: roots = (xmin, ) + roots
if right_root: roots = roots + (xmax, )
return roots
def makeWeights(
dataset,
plots=[],
database="weights.db",
compare=None, ## comparison function
delta=0.01, ## delta for ``mean'' weight variation
minmax=0.03, ## delta for ``minmax'' weight variation
power=None, ## auto-determination
debug=True, ## save intermediate information in DB
make_plots=False, ## make plots
tag="Reweighting"):
"""The main function: perform one re-weighting iteration
and reweight ``MC''-data set to looks as ``data''(reference) dataset
>>> results = makeWeights (
... dataset , ## data source to be reweighted (DataSet, TTree, abstract source)
... plots , ## reweighting plots
... database , ## datadabse to store/update reweigting results
... delta , ## stopping criteria for `mean` weight variation
... minmax , ## stopping criteria for `min/max` weight variation
... power , ## effective power to apply to the weigths
... debug = True , ## store debuig information in database
... make_plots = True , ## produce useful comparison plots
... tag = 'RW' ) ## tag for better printout
If `make_plots = False`, it returns the tuple of active reweitings:
>>> active = makeWeights ( ... , make_plots = False , ... )
Otherwise it also returns list of comparison plots
>>> active, cmp_plots = makeWeights ( ... , make_plots = True , ... )
>>> for item in cmp_plots :
... what = item.what
... hdata = item.data
... hmc = item.mc
... hweight = item.weight
If no more rewighting iteratios required, <code>active</code> is an empty tuple
"""
assert 0 < delta, "makeWeights(%s): Invalid value for ``delta'' %s" % (
tag, delta)
assert 0 < minmax, "makeWeights(%s): Invalid value for ``minmax'' %s" % (
tag, minmax)
from ostap.logger.colorized import allright, attention, infostr
from ostap.utils.basic import isatty
nplots = len(plots)
## if 1 < nplots :
## import math
## fudge_factor = math.sqrt ( 1.0 / max ( 2.0 , nplots - 1.0 ) )
## delta = delta * fudge_factor
## minmax = minmax * fudge_factor
## list of plots to compare
cmp_plots = []
## reweighting summary table
header = ('Reweighting', 'wmin/wmax', 'OK?', 'wrms[%]', 'OK?', 'chi2/ndf',
'ww', 'exp')
rows = {}
save_to_db = []
## number of active plots for reweighting
for wplot in plots:
what = wplot.what ## variable/function to plot/compare
how = wplot.how ## weight and/or additional cuts
address = wplot.address ## address in database
hdata0 = wplot.data ## original "DATA" object
hmc0 = wplot.mc_histo ## original "MC" histogram
ww = wplot.w ## relative weight
projector = wplot.projector ## projector for MC data
ignore = wplot.ignore ## ignore for weigtht building?
#
# normalize the data
#
hdata = hdata0
if isinstance(hdata, ROOT.TH1): hdata = hdata.density()
# =====================================================================
## make a plot on (MC) data with the weight
# =====================================================================
hmc0 = projector(dataset, hmc0, what, how)
st = hmc0.stat()
mnmx = st.minmax()
if iszero(mnmx[0]):
logger.warning("%s: statistic goes to zero %s/``%s''" %
(tag, st, address))
elif mnmx[0] <= 0:
logger.warning("%s: statistic is negative %s/``%s''" %
(tag, st, address))
# =====================================================================
## normalize MC
# =====================================================================
hmc = hmc0.density()
# =====================================================================
## calculate the reweighting factor : a bit conservative (?)
# this is the only important line
# =====================================================================
# try to exploit finer binning if/when possible
hboth = isinstance(hmc, ROOT.TH1) and isinstance(hdata, ROOT.TH1)
if hboth and 1 == hmc.dim () and 1 == hdata.dim () and \
len ( hmc ) >= len( hdata ) :
w = (1.0 / hmc) * hdata ## NB!
elif hboth and 2 == hmc.dim () and 2 == hdata.dim () and \
( hmc.binsx() >= hdata.binsx() ) and \
( hmc.binsy() >= hdata.binsy() ) :
w = (1.0 / hmc) * hdata ## NB!
elif hboth and 3 == hmc.dim () and 3 == hdata.dim () and \
( hmc.binsx() >= hdata.binsx() ) and \
( hmc.binsy() >= hdata.binsy() ) and \
( hmc.binsz() >= hdata.binsz() ) :
w = (1.0 / hmc) * hdata ## NB!
else:
w = hdata / hmc ## NB!
# =====================================================================
## scale & get the statistics of weights
w /= w.stat().mean().value()
cnt = w.stat()
#
mnw, mxw = cnt.minmax()
wvar = cnt.rms() / cnt.mean()
good1 = wvar.value() <= delta
good2 = abs(mxw - mnw) <= minmax
good = good1 and good2 ## small variance?
#
c2ndf = 0
for i in w:
c2ndf += w[i].chi2(1.0)
c2ndf /= (len(w) - 1)
## build the row in the summary table
row = address , \
'%-5.3f/%5.3f' % ( cnt.minmax()[0] , cnt.minmax()[1] ) , \
allright ( '+' ) if good2 else attention ( '-' ) , \
(wvar * 100).toString('%6.2f+-%-6.2f') , \
allright ( '+' ) if good1 else attention ( '-' ) , '%6.2f' % c2ndf
## make plots at the start of each iteration?
if make_plots:
item = ComparisonPlot(what, hdata, hmc, w)
cmp_plots.append(item)
row = tuple(list(row) + ['%4.3f' % ww if 1 != ww else ''])
rows[address] = row
#
## make decision based on the variance of weights
#
mnw, mxw = cnt.minmax()
if (not good) and (not ignore): ## small variance?
save_to_db.append((address, ww, hdata0, hmc0, hdata, hmc, w))
# =====================================================================
## make a comparison (if needed)
# =====================================================================
if compare: compare(hdata0, hmc0, address)
active = tuple([p[0] for p in save_to_db])
nactive = len(active)
if power and callable(power):
eff_exp = power(nactive)
elif isinstance(power, num_types) and 0 < power <= 1.5:
eff_exp = 1.0 * power
elif 1 == nactive and 1 < len(plots):
eff_exp = 0.95
elif 1 == nactive:
eff_exp = 1.00
else:
eff_exp = 1.10 / max(nactive, 1)
while database and save_to_db:
entry = save_to_db.pop()
address, ww, hd0, hm0, hd, hm, weight = entry
cnt = weight.stat()
mnw, mxw = cnt.minmax()
## avoid too large or too small weights
for i in weight:
w = weight[i]
if w.value() < 0.5:
weight[i] = VE(0.5, w.cov2())
elif w.value() > 2.0:
weight[i] = VE(2.0, w.cov2())
if 1 < nactive and 1 != ww:
eff_exp *= ww
logger.info("%s: apply ``effective exponent'' of %.3f for ``%s''" %
(tag, eff_exp, address))
if 1 != eff_exp and 0 < eff_exp:
weight = weight**eff_exp
row = list(rows[address])
row.append('%4.3f' % eff_exp)
rows[address] = tuple(row)
with DBASE.open(database) as db:
db[address] = db.get(address, []) + [weight]
if debug:
addr = address + ':REWEIGHTING'
db[addr] = db.get(addr, []) + list(entry[2:])
del hd0, hm0, hd, hm, weight, entry
table = [header]
for row in rows:
table.append(rows[row])
import ostap.logger.table as Table
logger.info(
'%s, active:#%d \n%s ' %
(tag, nactive,
Table.table(table, title=tag, prefix='# ', alignment='lccccccc')))
cmp_plots = tuple(cmp_plots)
return (active, cmp_plots) if make_plots else active
]
for o in functions:
fun = o[0] ## function
der = o[1] ## derivative
x = o[2] ## argument
logger.info(80 * '*')
for i in range(1, 6):
res = derivative(fun, x, 1.e-20, i, err=True)
f1 = res
d = der(x) ## the exact value for derivative
if iszero(d):
logger.info('Rule=%2d %s %s %s' % (2 * i + 1, f1, d,
(f1.value() - d)))
else:
logger.info('Rule=%2d %s %s %s' % (2 * i + 1, f1, d,
(f1.value() - d) / d))
logger.info(80 * '*')
## the function
func = math.sin
## analysitical derivative
deriv_a = math.cos
## numerical first derivative
deriv_1 = Derivative(func, order=5)
## numerical second derivative
def makeWeights ( dataset ,
plots = [] ,
database = "weights.db" ,
compare = None , ## comparison function
delta = 0.001 , ## delta for ``mean'' weigth variation
minmax = 0.05 , ## delta for ``minmax'' weigth variation
power = 0 , ## auto-determination
debug = True ) : ## save intermediate information in DB
assert 0 < delta , "Reweighting: Invalid value for ``delta'' %s" % delta
assert 0 < minmax , "Reweighting: Invalid value for ``minmax'' %s" % minmax
power = power if power >= 1 else len ( plots )
nplots = len ( plots )
if 1 < nplots :
import math
fudge_factor = math.sqrt ( 1.0 / max ( 2.0 , nplots - 1.0 ) )
delta = delta * fudge_factor
minmax = minmax * fudge_factor
save_to_db = []
## number of active plots for reweighting
active = 0
## loop over plots
for wplot in plots :
what = wplot.what ## variable/function to plot/compare
how = wplot.how ## weight and/or additional cuts
address = wplot.address ## address in database
hdata0 = wplot.data ## original "DATA" object
hmc0 = wplot.mc_histo ## original "MC" histogram
ww = wplot.w ## relative weight
#
# normailze the data
#
hdata = hdata0
if isinstance ( hdata , ROOT.TH1 ) : hdata = hdata.density ()
#
## make a plot on (MC) data with the weight
#
dataset.project ( hmc0 , what , how )
st = hmc0.stat()
mnmx = st.minmax()
if iszero ( mnmx[0] ) :
logger.warning ( "Reweighting: statistic goes to zero %s/``%s''" % ( st , address ) )
#
## normalize MC
#
hmc = hmc0.density()
#
## calculate the reweighting factor : a bit conservative (?)
#
# this is the only important line
#
# try to exploit finer binning if possible
if len ( hmc ) >= len( hdata ) : w = ( 1.0 / hmc ) * hdata ## NB!
else : w = hdata / hmc ## NB!
## scale & get the statistics of weights
w /= w.stat().mean().value()
cnt = w.stat()
#
wvar = cnt.rms()/cnt.mean()
logger.info ( 'Reweighting: %24s: mean/(min,max):%20s/(%.3f,%.3f) RMS:%s[%%]' %
( "``" + address + "''" ,
cnt.mean().toString('(%.2f+-%.2f)') ,
cnt.minmax()[0] ,
cnt.minmax()[1] , (wvar * 100).toString('(%.2f+-%.2f)') ) )
#
## make decision based on the variance of weights
#
mnw , mxw = cnt.minmax()
if wvar.value() <= delta and abs ( mxw - mnw ) <= minmax : ## small variance?
logger.info("Reweighting: No more reweights for ``%s'' [%.2f%%]/[(%+.1f,%+.1f)%%]" % \
( address , wvar * 100 , ( mnw - 1 ) * 100 , ( mxw - 1 ) * 100 ) )
del w , hdata , hmc
else :
save_to_db.append ( ( address , ww , hdata0 , hmc0 , hdata , hmc , w ) )
#
## make a comparison (if needed)
#
if compare : compare ( hdata0 , hmc0 , address )
## for single reweighting
if 1 == nplots : power = 1
if power != nplots :
logger.info ( "Reweighting: ``power'' is %g/#%d" % ( power , nplots ) )
active = len ( save_to_db )
if active != nplots :
logger.info ( "Reweighting: number of ``active'' reweights %s/#%d" % ( active , nplots ) )
if database and save_to_db :
power += ( nplots - active )
logger.info ("Reweighting: ``power'' is changed to %g" % power )
while database and save_to_db :
entry = save_to_db.pop()
address, ww , hd0, hm0, hd , hm , weight = entry
eff_exp = 1.0 / power
if 1 != nplots and 1 != ww :
eff_exp *= ww
logger.info ("Reweighting: apply ``effective exponent'' of %.3f for ``%s''" % ( eff_exp , address ) )
if 1 != eff_exp and 0 < eff_exp :
weight = weight ** eff_exp
## print 'WEIGHT stat', eff_exp, weight.stat()
## hmmmm... needed ? yes!
#if 1 < power : weight = weight ** ( 1.0 / power )
## relative importance
#if 1 != ww :
# logger.info ("Reweighting: apply ``relative importance factor'' of %.3g for ``'%s'" % ( ww , address ) )
# weight = weight ** ww
with DBASE.open ( database ) as db :
db[address] = db.get( address , [] ) + [ weight ]
if debug :
addr = address + ':REWEIGHTING'
db [ addr ] = db.get ( addr , [] ) + list ( entry[2:] )
del hd0, hm0 , hd , hm , weight , entry
return active