def test_correlations_of_uparams_and_derivatives():
T = 10
size = 100
params = {'alpha': 0.17, 'beta': 1.18}
data = gen.bgext_model(T, params['alpha'], params['beta'], size=size)
data = compress_bgext_data(data)
model = models.BGModel(penalizer_coef=0.1)
model.fit(data['frequency'],
data['T'],
bootstrap_size=10,
N=data['N'],
initial_params=params.values())
print("Generation params")
print(params)
print("Fitted params")
print(model.params)
print(model.params_C)
print("Uncertain parameters")
print(model.uparams)
assert is_almost_equal(
correlation_matrix([model.uparams['alpha'],
model.uparams['alpha']])[0, 1], 1.0)
assert 1.0 > correlation_matrix([
model.uparams['alpha'] + ufloat(1, 1), model.uparams['alpha']
])[0, 1] > 0.0
# stub of profile
p1 = model.expected_number_of_purchases_up_to_time(1)
p2 = model.expected_number_of_purchases_up_to_time(2)
assert 1.0 > correlation_matrix([p1, p2])[0, 1] > 0.0
# stub of profile
p1 = model.expected_number_of_purchases_up_to_time(1)
p2 = model.expected_number_of_purchases_up_to_time(10)
assert 1.0 > correlation_matrix([p1, p2])[0, 1] > 0.0
# stub of profile
p1 = model.expected_number_of_purchases_up_to_time(1)
p2 = model.expected_number_of_purchases_up_to_time(100)
assert 1.0 > correlation_matrix([p1, p2])[0, 1] > 0.0
def calculation_Q_conventional(K, a):
if kontrola_rozmeru(K, a)==False:
return False, False, False
Q = -np.dot(K, a)
Q_covarianceMatrix = covariance_matrix(Q)
Q_correlationMatrix = correlation_matrix(Q)
return Q, Q_covarianceMatrix, Q_correlationMatrix
def get_covariance_matrix(self):
'''
Get the covariance matrix from all contributions
https://root.cern.ch/doc/master/classTUnfoldDensity.html#a7f9335973b3c520e2a4311d2dd6f5579
'''
import uncertainties as u
from numpy import array, matrix, zeros
from numpy import sqrt as np_sqrt
if self.unfolded_data is not None:
# Calculate the covariance from TUnfold
covariance = asrootpy(
self.unfoldObject.GetEmatrixInput('Covariance'))
# Reformat into a numpy matrix
zs = list(covariance.z())
cov_matrix = matrix(zs)
# Just the unfolded number of events
inputs = hist_to_value_error_tuplelist(self.unfolded_data)
nominal_values = [i[0] for i in inputs]
# # Unfolded number of events in each bin
# # With correlation between uncertainties
values_correlated = u.correlated_values( nominal_values, cov_matrix.tolist() )
corr_matrix = matrix(u.correlation_matrix(values_correlated) )
return cov_matrix, corr_matrix
else:
print("Data has not been unfolded. Cannot return unfolding covariance matrix")
return
def test_uncertainties_comparison_user_defined_funcs():
import uncertainties
from uncertainties import ufloat
x = UQ_( '15. +/- 0.1 m' )
y = UQ_( '25. +/- 0.2 m' )
w = UQ_( '35. +/- 0.3 m' )
xx = ufloat( 15, 0.1 )
yy = ufloat( 25, 0.2 )
ww = ufloat( 35, 0.3 )
def calc(x,y,w):
return ((x - y)/(x - w))*w
f = uconv.WithError(calc)
ff = uncertainties.wrap(calc)
z = f(x,y,w)
zz = ff(xx,yy,ww)
t_us = timeit.timeit( lambda : f( x, y, w), number = 200 )
t_unc = timeit.timeit( lambda : ff(xx,yy,ww), number = 200 )
assert t_us > t_unc
assert Close( z.nominal.magnitude , zz.nominal_value)
assert Close( z.uncertainty.magnitude, zz.std_dev)
corr = uconv.correlations.matrix( x,y,z,w )
ccorr = uncertainties.correlation_matrix( [xx,yy,zz,ww] )
for i in range(4):
for j in range(4):
assert Close( corr(i,j), ccorr[i][j], 0.1 )
def test_uncertainties_comparison_speed():
import uncertainties
from uncertainties import ufloat
x = UQ_( '15. +/- 0.1 m' )
y = UQ_( '25. +/- 0.2 m' )
w = UQ_( '35. +/- 0.3 m' )
xx = ufloat( 15, 0.1 )
yy = ufloat( 25, 0.2 )
ww = ufloat( 35, 0.3 )
z = ((x - y)/(x - w))*w
zz = ((xx - yy)/(xx - ww))*ww
t_us = timeit.timeit( lambda : (((x - y)/(x - w))*w).uncertainty , number = 200 )
t_unc = timeit.timeit( lambda : (((xx - yy)/(xx - ww))*ww).std_dev, number = 200 )
assert t_us > t_unc
assert Close( z.nominal.magnitude , zz.nominal_value)
assert Close( z.uncertainty.magnitude, zz.std_dev)
corr = uconv.correlations.matrix( x,y,z,w )
ccorr = uncertainties.correlation_matrix( [xx,yy,zz,ww] )
for i in range(4):
for j in range(4):
assert Close( corr(i,j), ccorr[i][j], 0.1 )
def calculate_normalised_xsection(inputs,
bin_widths,
normalise_to_one=False,
covariance_matrix=None,
inputMC_covariance_matrix=None):
"""
Calculates normalised average x-section for each bin: 1/N *1/bin_width sigma_i
There are two ways to calculate this
1) N = sum(sigma_i)
2) N = sum(sigma_i/bin_width)
The latter one will normalise the total distribution to 1
@param inputs: list of value-error pairs
@param bin_widths: bin widths of the inputs
"""
values = [u.ufloat(i[0], i[1]) for i in inputs]
norm_cov_matrix = None
norm_corr_matrix = None
inputMC_norm_cov_matrix = None
if not covariance_matrix is None and not normalise_to_one:
values_correlated = u.correlated_values(
[v.nominal_value for v in values], covariance_matrix.tolist())
norm = sum(values_correlated)
norm_values_correlated = []
# Loop over unfolded number of events with correlated uncertainties
# And corresponding bin width
# Calculate normalised cross section, with correctly correlated uncertainty
for v, width in zip(values_correlated, bin_widths):
norm_values_correlated.append(v / width / norm)
# Get covariance and correlation matrix for normalised cross section
norm_cov_matrix = matrix(u.covariance_matrix(norm_values_correlated))
norm_corr_matrix = matrix(u.correlation_matrix(norm_values_correlated))
result = [(v.nominal_value, v.std_dev) for v in norm_values_correlated]
# Get Covariance Matrix for input MC
if not inputMC_covariance_matrix is None:
inputMC_values_correlated = u.correlated_values(
[v.nominal_value for v in values],
inputMC_covariance_matrix.tolist())
inputMC_norm = sum(inputMC_values_correlated)
inputMC_norm_values_correlated = []
for v, width in zip(inputMC_values_correlated, bin_widths):
inputMC_norm_values_correlated.append(v / width / inputMC_norm)
inputMC_norm_cov_matrix = matrix(
u.covariance_matrix(inputMC_norm_values_correlated))
else:
normalisation = 0
if normalise_to_one:
normalisation = sum([
value / bin_width
for value, bin_width in zip(values, bin_widths)
])
else:
normalisation = sum(values)
xsections = [(1 / bin_width) * value / normalisation
for value, bin_width in zip(values, bin_widths)]
result = [(xsection.nominal_value, xsection.std_dev)
for xsection in xsections]
return result, norm_cov_matrix, norm_corr_matrix, inputMC_norm_cov_matrix
def test_BGBB_correlations_preserved():
T = 10
size = 100
params = {'alpha': 1.2, 'beta': 0.7, 'gamma': 0.6, 'delta': 2.7}
data = gen.bgbb_model(T,
params['alpha'],
params['beta'],
params['gamma'],
params['delta'],
size=size)
data = compress_data(data)
model = models.BGBBModel()
model.fit(data['frequency'],
data['recency'],
data['T'],
bootstrap_size=10,
N=data['N'],
initial_params=params.values())
print("Generation params")
print(params)
print("Fitted params")
print(model.params)
print(model.params_C)
print("Uncertain parameters")
print(model.uparams)
assert is_almost_equal(
correlation_matrix([model.uparams['alpha'],
model.uparams['alpha']])[0, 1], 1.0)
assert 1.0 > correlation_matrix([
model.uparams['alpha'] + ufloat(1, 1), model.uparams['alpha']
])[0, 1] > 0.0
# stub of profile
p1 = model.expected_number_of_purchases_up_to_time(1)
p2 = model.expected_number_of_purchases_up_to_time(2)
assert 1.0 > correlation_matrix([p1, p2])[0, 1] > 0.0
def calculation_Q_conventional(K, a_out, a):
#pridani vnejsi koncentrace do vektoru koncentraci
a = np.append(a, ufloat(a_out, 0.05 * a_out))
if kontrola_rozmeru(K, a) == False:
return False, False, False
Q = -np.dot(K, a)
Q_covarianceMatrix = covariance_matrix(Q)
Q_correlationMatrix = correlation_matrix(Q)
return Q, Q_covarianceMatrix, Q_correlationMatrix
def calculation_Q_conventional():
K, a = prepare_for_calculation()
# breakpoint()
if np.isscalar(K):
print('Nelze provest vypocet.')
Q = -np.dot(K, a)
Q_covarianceMatrix = covariance_matrix(Q)
Q_correlationMatrix = correlation_matrix(Q)
return Q, Q_covarianceMatrix, Q_correlationMatrix
def calculate_xsection(inputs,
bin_widths,
luminosity,
efficiency=1.,
covariance_matrix=None,
inputMC_covariance_matrix=None):
'''
BUG: this doesn't work unless the inputs are unfolded!
inputs = list of value-error pairs
luminosity = integrated luminosity of the measurement
'''
abs_cov_matrix = None
abs_corr_matrix = None
inputMC_abs_cov_matrix = None
result = []
add_result = result.append
values = [u.ufloat(i[0], i[1]) for i in inputs]
if not covariance_matrix is None:
values_correlated = u.correlated_values(
[v.nominal_value for v in values], covariance_matrix.tolist())
abs_values_correlated = []
# Loop over unfolded number of events with correlated uncertainties
# And corresponding bin width
# Calculate absolute cross section, with correctly correlated uncertainty
for v, width in zip(values_correlated, bin_widths):
abs_values_correlated.append(v / width / luminosity / efficiency)
# Get covariance and correlation matrix for absolute cross section
abs_cov_matrix = matrix(u.covariance_matrix(abs_values_correlated))
abs_corr_matrix = matrix(u.correlation_matrix(abs_values_correlated))
result = [(v.nominal_value, v.std_dev) for v in abs_values_correlated]
# Get Covariance Matrix for input MC
if not inputMC_covariance_matrix is None:
inputMC_values_correlated = u.correlated_values(
[v.nominal_value for v in values],
inputMC_covariance_matrix.tolist())
inputMC_abs_values_correlated = []
for v, width in zip(inputMC_values_correlated, bin_widths):
inputMC_abs_values_correlated.append(v / width / luminosity /
efficiency)
inputMC_abs_cov_matrix = matrix(
u.covariance_matrix(inputMC_abs_values_correlated))
else:
for valueAndErrors, binWidth in zip(inputs, bin_widths):
value = valueAndErrors[0]
error = valueAndErrors[1]
xsection = value / (luminosity * efficiency * binWidth)
xsection_error = error / (luminosity * efficiency * binWidth)
add_result((xsection, xsection_error))
return result, abs_cov_matrix, abs_corr_matrix, inputMC_abs_cov_matrix
def get_covariance_matrix(self):
'''
Get the covariance matrix from all contributions
https://root.cern.ch/doc/master/classTUnfoldDensity.html#a7f9335973b3c520e2a4311d2dd6f5579
'''
import uncertainties as u
from numpy import array, matrix, zeros
from numpy import sqrt as np_sqrt
if self.unfolded_data is not None:
# Get the statistical data covariance from TUnfold
covariance = asrootpy(
self.unfoldObject.GetEmatrixInput('Covariance'))
# Reformat into a numpy matrix
zs = list(covariance.z())
cov_matrix = matrix(zs)
# Just the unfolded number of events
inputs = hist_to_value_error_tuplelist(self.unfolded_data)
nominal_values = [i[0] for i in inputs]
# # Unfolded number of events in each bin
# # With correlation between uncertainties
values_correlated = u.correlated_values( nominal_values, cov_matrix.tolist() )
corr_matrix = matrix(u.correlation_matrix(values_correlated) )
# Get the input MC statisical covariance for the response from TUnfold
inputMC_stat_covariance = asrootpy(
self.unfoldObject.GetEmatrixSysUncorr('InputMC_Stat_Covariance'))
z = list(inputMC_stat_covariance.z())
inputMC_cov_matrix = matrix(z)
return cov_matrix, corr_matrix, inputMC_cov_matrix
else:
print("Data has not been unfolded. Cannot return unfolding covariance matrix")
return
#print corr_matrix #mm = unc.ufloat(1.25, 1.e-9,) #at = unc.ufloat(0.51, 1.e-9,) #qcd = mm-(1.5+(at-0.5)*10.) #qcd = mm-(1.5+(at-0.5)*10.) #c_matrix = unc.correlation_matrix([qcd,mm,at]) #print c_matrix c1 = unc.ufloat(1.5, 1.e-9,) c2 = unc.ufloat(0.5, 1.e-9,) c3 = unc.ufloat(10., 1.e-9,) c4 = unc.ufloat(1.06, 1.e-9,) at = unc.ufloat(0.51, 1.e-9,) mm=(c1+(at-c2)*c3) c_matrix = unc.correlation_matrix([mm,at]) print c_matrix #gaus = 0.75*math.exp(-0.5*pow((mm.n-(1.5+(at.n-0.5)*5.))/1.06,2.)) #exp = math.exp(70.0-120.0*at.n)*580. #qcd = gaus*exp #c_matrix = unc.correlation_matrix([qcd,gaus,exp]) #print c_matrix #(u2, v2, sum2) = unc.correlated_values([u.n, v.n, sum_value.n], cov_matrix) #print u2, v2, sum2 #print sum2 - (u2+2*v2) #print unc.covariance_matrix([u2, v2, sum2]) # #(u3, v3, sum3) = unc.correlated_values_norm( [u, v, sum_value], corr_matrix)
def test_BGBBBGExt_integration_in_models_with_uncertainties():
T = 10
size = 100
params = {'alpha': 1.2, 'beta': 0.7, 'gamma': 0.6, 'delta': 2.7, 'epsilon': 1.0, 'zeta': 10.0, 'c0': 0.05}
data = gen.bgbbbgext_model(T, params['alpha'], params['beta'], params['gamma'], params['delta'],
params['epsilon'],
params['zeta'], params['c0'], size=size, time_first_purchase=True)
compressed_data = compress_session_session_before_conversion_data(data)
model = mod.BGBBBGExtModel(penalizer_coef=0.2)
model.fit(frequency=compressed_data['frequency'], recency=compressed_data['recency'], T=compressed_data['T'],
frequency_before_conversion=compressed_data['frequency_before_conversion'],
N=compressed_data['N'], initial_params=params.values())
print("Generation params")
print(params)
print("Fitted params")
print(model.params)
print(model.params_C)
print("Uncertain parameters")
print(model.uparams)
# test correlations preserved
assert is_almost_equal(correlation_matrix([model.uparams['alpha'], model.uparams['alpha']])[0, 1], 1.0)
assert 1.0 > correlation_matrix([model.uparams['alpha'] + ufloat(1, 1), model.uparams['alpha']])[0, 1] > 0.0
# stub of profile
p1 = model.expected_number_of_sessions_up_to_time(1)
p2 = model.expected_number_of_sessions_up_to_time(2)
assert 1.0 > correlation_matrix([p1, p2])[0, 1] > 0.0
print("E[X(t)] as a function of t")
for t in [0, 1, 10, 100, 1000, 10000]:
uEx = model.expected_number_of_sessions_up_to_time(t)
print(t, uEx)
assert uEx.n >= 0
assert uEx.s >= 0
t = 10
print("E[X(t) = n] as a function of n, t = " + str(t))
tot_prob = 0.0
for n in range(t + 1):
prob = model.fitter.probability_of_n_sessions_up_to_time(t, n)
print(n, prob)
tot_prob += prob
assert 1 >= prob >= 0
uprob = model.probability_of_n_sessions_up_to_time(t, n)
print(uprob)
assert is_almost_equal(uprob.n, prob)
assert math.fabs(tot_prob - 1.0) < 0.00001
print("c(t) as a function of t")
for t in [0, 1, 10, 100, 1000]:
uc = model.expected_probability_of_converting_at_time(t)
print(t, uc)
assert uc.n >= 0.0 and uc.n <= 1.0
assert uc.s >= 0.0
print("cumulative c(t) as a function of t")
for t in [0, 1, 2, 3, 4, 5, 7, 10, 20, 50, 100]:
uc = model.expected_probability_of_converting_within_time(t)
print(t, uc)
assert uc.n >= 0.0 and uc.n <= 1.0
assert uc.s >= 0.0
def test_uncertainties_comparison_general():
import uncertainties
from uncertainties import ufloat
# compare error propagation.
x = UQ_( '2.5 +/- 0.5 m' )
y = UQ_( '2.5 +/- 0.5 m' )
w = x
xx = ufloat( 2.5, 0.5 )
yy = ufloat( 2.5, 0.5 )
ww = xx
z = x+y
zz = xx+yy
assert Close( z.nominal.magnitude, zz.nominal_value, 1e-5 )
assert Close( z.uncertainty.magnitude, zz.std_dev, 1e-5 )
z = x-y/2
zz = xx-yy/2
assert Close( z.nominal.magnitude, zz.nominal_value, 1e-5 )
assert Close( z.uncertainty.magnitude, zz.std_dev, 1e-5 )
z = x*y
zz = xx*yy
assert Close( z.nominal.magnitude, zz.nominal_value, 1e-5 )
assert Close( z.uncertainty.magnitude, zz.std_dev, 1e-5 )
z = x/y
zz = xx/yy
assert Close( z.nominal.magnitude, zz.nominal_value, 1e-5 )
# linear approximation differs here!
assert not Close( z.uncertainty.magnitude, zz.std_dev, 1e-5 )
z = w - x
zz = ww - xx
assert Close( z.nominal.magnitude, 0, 1e-5 )
assert Close( z.nominal.magnitude, zz.nominal_value, 1e-5 )
assert Close( z.uncertainty.magnitude, zz.std_dev, 1e-5 )
# add correlation
x.correlated(y,1)
(xx,yy) = uncertainties.correlated_values_norm( [(2.5,0.5), (2.5,0.5)], [ [1,1],[1,1] ] )
z = x - y
zz = xx - yy
assert Close( z.nominal.magnitude, 0, 1e-5 )
assert Close( z.nominal.magnitude, zz.nominal_value, 1e-5 )
assert Close( z.uncertainty.magnitude, zz.std_dev, 1e-5 )
num = 2
data = [ UQ_('2 +/- 0.1 m') ] * num
ddata = [ ufloat(2,0.1) ] * num
z = sum(data)
zz = sum(ddata)
assert Close( z.nominal.magnitude, 2*2, 1e-5 )
assert Close( z.uncertainty.magnitude, (2*0.1), 1e-5 )
assert Close( z.nominal.magnitude, zz.nominal_value, 1e-5 )
assert Close( z.uncertainty.magnitude, zz.std_dev, 1e-5 )
num = 10
data = [ UQ_('2 +/- 0.1 m') ] * num
ddata = [ ufloat(2,0.1) ] * num
z = sum(data)
zz = sum(ddata)
assert Close( z.nominal.magnitude, 10*2, 1e-5 )
assert Close( z.uncertainty.magnitude, (10*0.1), 1e-5 )
assert Close( z.nominal.magnitude, zz.nominal_value, 1e-5 )
assert Close( z.uncertainty.magnitude, zz.std_dev, 1e-5 )
zz = 2*xx
ww = zz + yy
z = 2*x
w = z + y
corr = uncertainties.correlation_matrix( [xx,yy,zz,ww] )
assert x.correlation(x) == corr[0][0]
assert x.correlation(y) == corr[0][1]
assert x.correlation(z) == corr[0][2]
assert x.correlation(w) == corr[0][3]
assert x.correlation(x) == corr[0][0]
assert y.correlation(x) == corr[1][0]
assert z.correlation(x) == corr[2][0]
assert w.correlation(x) == corr[3][0]