class EstStats(ExpSwitcher):
'''
A simple agent that estimates various statistics
of the observations.
'''
def init(self, boot_spec):
ExpSwitcher.init(self, boot_spec)
if len(boot_spec.get_observations().shape()) != 1:
raise UnsupportedSpec('I assume 1D signals.')
self.y_stats = MeanCovariance()
def merge(self, other):
self.y_stats.merge(other.y_stats)
def process_observations(self, obs):
y = obs['observations']
dt = obs['dt'].item()
self.y_stats.update(y, dt)
def get_state(self):
return dict(y_stats=self.y_stats)
def set_state(self, state):
self.y_stats = state['y_stats']
def publish(self, pub):
if self.y_stats.get_num_samples() == 0:
pub.text('warning', 'Too early to publish anything.')
return
Py = self.y_stats.get_covariance()
Ry = self.y_stats.get_correlation()
Py_inv = self.y_stats.get_information()
Ey = self.y_stats.get_mean()
y_max = self.y_stats.get_maximum()
y_min = self.y_stats.get_minimum()
Ry0 = Ry.copy()
np.fill_diagonal(Ry0, np.NaN)
Py0 = Py.copy()
np.fill_diagonal(Py0, np.NaN)
pub.text('stats', 'Num samples: %s' % self.y_stats.get_num_samples())
with pub.plot('y_bounds') as pylab:
style_ieee_fullcol_xy(pylab)
pylab.plot(Ey, label='E(y)')
pylab.plot(y_max, label='y_max')
pylab.plot(y_min, label='y_min')
pylab.legend()
all_positive = (np.min(Ey) > 0
and np.min(y_max) > 0
and np.min(y_min) > 0)
if all_positive:
with pub.plot('y_stats_log') as pylab:
style_ieee_fullcol_xy(pylab)
pylab.semilogy(Ey, label='E(y)')
pylab.semilogy(y_max, label='y_max')
pylab.semilogy(y_min, label='y_min')
pylab.legend()
pub.array_as_image('Py', Py, caption='cov(y)')
pub.array_as_image('Py0', Py0, caption='cov(y) - no diagonal')
pub.array_as_image('Ry', Ry, caption='corr(y)')
pub.array_as_image('Ry0', Ry0, caption='corr(y) - no diagonal')
pub.array_as_image('Py_inv', Py_inv)
pub.array_as_image('Py_inv_n', cov2corr(Py_inv))
with pub.plot('Py_svd') as pylab: # XXX: use spectrum
style_ieee_fullcol_xy(pylab)
_, s, _ = np.linalg.svd(Py)
s /= s[0]
pylab.semilogy(s, 'bx-')
with pub.subsection('y_stats') as sub:
self.y_stats.publish(sub)
class BDSEEstimator(BDSEEstimatorInterface):
"""
Estimates a BDSE model.
Tensors used: ::
M^s_vi (N) x (N x K)
N^s_i (N) x (K)
T^svi (NxNxK)
U^si (NxK)
"""
@contract(rcond='float,>0')
def __init__(self, rcond=1e-10, antisym_T=False, antisym_M=False, use_P_scaling=False):
"""
:param rcond: Threshold for computing pseudo-inverse of P.
:param antisym_T: If True, the estimate of T is antisymmetrized.
:param antisym_M: If True, the estimate of M is antisymmetrized.
"""
self.rcond = rcond
self.antisym_M = antisym_M
self.antisym_T = antisym_T
self.use_P_scaling = use_P_scaling
self.info('rcond: %f' % rcond)
self.info('antisym_T: %s' % antisym_T)
self.info('antisym_M: %s' % antisym_M)
self.info('use_P_scaling: %s' % use_P_scaling)
self.T = Expectation()
self.U = Expectation()
self.y_stats = MeanCovariance()
self.u_stats = MeanCovariance()
self.nsamples = 0
self.once = False
def merge(self, other):
assert isinstance(other, BDSEEstimator)
self.T.merge(other.T)
self.U.merge(other.U)
self.y_stats.merge(other.y_stats)
self.u_stats.merge(other.u_stats)
self.nsamples += other.nsamples
@contract(u='array[K],K>0,finite',
y='array[N],N>0,finite',
y_dot='array[N],finite', w='>0')
def update(self, y, u, y_dot, w=1.0):
self.once = True
self.nsamples += 1
self.n = y.size
self.k = u.size # XXX: check
self.y_stats.update(y, w)
self.u_stats.update(u, w)
# remove mean
u_n = u - self.u_stats.get_mean()
y_n = y - self.y_stats.get_mean()
# make products
T_k = outer(outer(y_n, y_dot), u_n)
assert T_k.shape == (self.n, self.n, self.k)
U_k = outer(y_dot, u_n)
assert U_k.shape == (self.n, self.k)
# update tensor
self.T.update(T_k, w)
self.U.update(U_k, w)
def get_P_inv_cond(self):
P = self.y_stats.get_covariance()
if False:
P_inv = np.linalg.pinv(P, rcond=self.rcond)
if True:
P2 = P + np.eye(P.shape[0]) * self.rcond
P_inv = np.linalg.inv(P2)
return P_inv
def get_T(self):
T = self.T.get_value()
if self.antisym_T:
self.info('antisymmetrizing T')
T = antisym(T)
return T
def get_model(self):
T = self.get_T()
U = self.U.get_value()
P = self.y_stats.get_covariance()
Q = self.u_stats.get_covariance()
P_inv = self.get_P_inv_cond()
Q_inv = np.linalg.pinv(Q)
if False:
M = get_M_from_P_T_Q(P, T, Q)
else:
if hasattr(self, 'use_P_scaling') and self.use_P_scaling:
M = get_M_from_P_T_Q_alt_scaling(P, T, Q)
else:
warnings.warn('untested')
try:
M = get_M_from_Pinv_T_Q(P_inv, T, Q)
except LinAlgError as e:
msg = 'Could not get_M_from_Pinv_T_Q.\n'
msg += indent(traceback.format_exc(e), '> ')
raise BDSEEstimatorInterface.NotReady(msg)
UQ_inv = np.tensordot(U, Q_inv, axes=(1, 0))
# This works but badly conditioned
Myav = np.tensordot(M, self.y_stats.get_mean(), axes=(1, 0))
N = UQ_inv - Myav
if self.antisym_M:
self.info('antisymmetrizing M')
M = antisym(M)
# # Note: this does not work, don't know why
# if False:
# printm('MYav1', Myav)
# y2 = np.linalg.solve(P, self.y_stats.get_mean())
# Myav2 = np.tensordot(T, y2, axes=(0, 0))
# # Myav = np.tensordot(T, y2, axes=(1, 0))
# printm('MYav2', Myav2)
# if False:
# printm('U', U, 'Q_inv', Q_inv)
# printm('UQ_inv', UQ_inv, 'Myav', Myav, 'N', N)
# printm('u_mean', self.u_stats.get_mean())
# printm('u_std', np.sqrt(Q.diagonal()))
# printm('y_mean', self.y_stats.get_mean())
self.Myav = Myav
self.UQ_inv = UQ_inv
return BDSEmodel(M, N)
def publish(self, pub):
if not self.once:
pub.text('warning', 'not updated yet')
return
pub.text('nsamples', '%s' % self.nsamples)
pub.text('rcond', '%g' % self.rcond)
with pub.subsection('model') as sub:
try:
model = self.get_model()
model.publish(sub)
except BDSEEstimatorInterface.NotReady as e:
pub.text('not-ready', str(e))
with pub.subsection('tensors') as sub:
T = self.get_T()
U = self.U.get_value()
P = self.y_stats.get_covariance()
Q = self.u_stats.get_covariance()
P_inv = np.linalg.pinv(P)
P_inv_cond = self.get_P_inv_cond()
Q_inv = np.linalg.pinv(Q)
#
# TP_inv2 = obtain_TP_inv_from_TP_2(T, P)
# M2 = np.tensordot(TP_inv2, Q_inv, axes=(2, 0))
pub_tensor3_slice2(sub, 'T', T)
pub_tensor2_comp1(sub, 'U', U)
pub_tensor2_cov(sub, 'P', P, rcond=self.rcond)
pub_tensor2_cov(sub, 'P_inv', P_inv)
pub_tensor2_cov(sub, 'P_inv_cond', P_inv_cond)
pub_tensor2_cov(sub, 'Q', Q)
pub_tensor2_cov(sub, 'Q_inv', Q_inv)
# Might not have been computed
# pub_tensor2_comp1(sub, 'Myav', self.Myav)
# pub_tensor2_comp1(sub, 'UQ_inv', self.UQ_inv)
with pub.subsection('y_stats') as sub:
self.y_stats.publish(sub)
with pub.subsection('u_stats') as sub:
self.u_stats.publish(sub)
with pub.subsection('alternative', robust=True) as sub:
sub.text('info', 'This is estimating without conditioning P')
T = self.get_T()
P = self.y_stats.get_covariance()
Q = self.u_stats.get_covariance()
M1 = get_M_from_P_T_Q(P, T, Q)
pub_tensor3_slice2(sub, 'get_M_from_P_T_Q', M1)
M2 = get_M_from_P_T_Q_alt(P, T, Q)
pub_tensor3_slice2(sub, 'get_M_from_P_T_Q_alt', M2)
M3 = get_M_from_P_T_Q_alt_scaling(P, T, Q)
pub_tensor3_slice2(sub, 'get_M_from_P_T_Q_alt2', M3)
