def __init__(self, **other):
BDSEEstimator.__init__(self, **other)
self.T = ExpectationWeighted()
self.U = ExpectationWeighted()
self.y_mean = ExpectationWeighted() # XXX: not necessary, y_stats.get_mean()
self.y_stats = MeanCovariance() # TODO: make robust
self.u_stats = MeanCovariance()
self.once = False
def init(self, boot_spec):
ExpSwitcher.init(self, boot_spec)
if len(boot_spec.get_observations().shape()) != 1:
raise UnsupportedSpec('I assume 1D signals.')
self.y_stats = MeanCovariance()
self.y_dot_stats = MeanCovariance()
self.y_dot_sgn_stats = MeanCovariance()
self.y_dot_abs_stats = MeanCovariance()
self.count = 0
self.y_deriv = DerivativeBox()
def __init__(self, max_y_dot, max_gy, min_y, max_y):
self.min_y = min_y
self.max_y = max_y
self.max_y_dot = max_y_dot
self.max_gy = max_gy
self.w_stats = MeanCovariance()
warnings.warn('This is not invariant to linear transformation (max,min y).')
self.once = False
def __init__(self, rcond=1e-10, antisym_T=False, antisym_M=False, use_P_scaling=False):
"""
:param rcond: Threshold for computing pseudo-inverse of P.
:param antisym_T: If True, the estimate of T is antisymmetrized.
:param antisym_M: If True, the estimate of M is antisymmetrized.
"""
self.rcond = rcond
self.antisym_M = antisym_M
self.antisym_T = antisym_T
self.use_P_scaling = use_P_scaling
self.info('rcond: %f' % rcond)
self.info('antisym_T: %s' % antisym_T)
self.info('antisym_M: %s' % antisym_M)
self.info('use_P_scaling: %s' % use_P_scaling)
self.T = Expectation()
self.U = Expectation()
self.y_stats = MeanCovariance()
self.u_stats = MeanCovariance()
self.nsamples = 0
self.once = False
def init(self, boot_spec):
self.boot_spec = boot_spec
if len(boot_spec.get_observations().shape()) != 1:
raise UnsupportedSpec('This agent can only work with 1D signals.')
self.count = 0
self.rd = RemoveDoubles(self.change_fraction)
self.y_deriv = DerivativeBox()
self.bdse_estimator = instantiate_spec(self.estimator_spec)
if not isinstance(self.bdse_estimator, BDSEEstimatorInterface):
msg = ('Expected a BDSEEstimatorInterface, got %s'
% describe_type(self.estimator))
raise ValueError(msg)
self.y_stats = MeanCovariance()
self.explorer.init(boot_spec)
self.commands_spec = boot_spec.get_commands()
# All the rest are only statistics
self.stats = MiscStatistics()
class Embed(ExpSwitcher):
def __init__(self, statistic='y_corr', scale_score=False, **kwargs): # @UnusedVariable
ExpSwitcher.__init__(self, **kwargs)
self.statistic = statistic
self.scale_score = False
def init(self, boot_spec):
ExpSwitcher.init(self, boot_spec)
if len(boot_spec.get_observations().shape()) != 1:
raise UnsupportedSpec('I assume 1D signals.')
self.y_stats = MeanCovariance()
self.y_dot_stats = MeanCovariance()
self.y_dot_sgn_stats = MeanCovariance()
self.y_dot_abs_stats = MeanCovariance()
self.count = 0
self.y_deriv = DerivativeBox()
def get_similarity(self, which):
if which == 'y_corr':
return self.y_stats.get_correlation()
if which == 'y_dot_corr':
return self.y_dot_stats.get_correlation()
if which == 'y_dot_sgn_corr':
return self.y_dot_sgn_stats.get_correlation()
if which == 'y_dot_abs_corr':
return self.y_dot_abs_stats.get_correlation()
raise ValueError()
# check_contained(statistic, self.statistics, 'statistic')
def process_observations(self, obs):
y = obs['observations']
dt = obs['dt'].item()
self.y_deriv.update(y, dt)
if self.y_deriv.ready():
y, y_dot = self.y_deriv.get_value()
self.y_stats.update(y, dt)
self.y_dot_stats.update(y_dot, dt)
self.y_dot_sgn_stats.update(np.sign(y_dot), dt)
self.y_dot_abs_stats.update(np.abs(y_dot), dt)
self.count += 1
def get_S(self, dimensions=2, pub=None):
similarity = self.get_similarity(self.statistic)
if pub is not None:
pub.array_as_image('similarity', similarity,
caption='Similarity statistic')
plot_spectrum(pub, 'similarity', similarity)
if self.scale_score:
R = scale_score(similarity).astype('float32')
R = R / R.max()
if pub is not None:
pub.array_as_image('scale_score', R)
else:
R = similarity
D = 1 - R
D = D * np.pi / D.max()
np.fill_diagonal(D, 0)
if pub is not None:
# pub.array_as_image('D', D)
P = D * D
B = double_center(P)
# plot_spectrum(pub, 'D', D)
# plot_spectrum(pub, 'P', P)
plot_spectrum(pub, 'B', B)
S = mds(D, ndim=dimensions)
# S = inner_product_embedding(similarity, 3)
# S = S[1:3, :]
return S
def get_S_discrete(self, dimensions=2, pub=None):
R = self.y_dot_abs_stats.get_correlation()
Dis = discretize(-R, 2)
np.fill_diagonal(Dis, 0)
R = R * R
C = np.maximum(R, 0)
if pub is not None:
pub.array_as_image('Dis', Dis)
pub.array_as_image('R', R)
pub.array_as_image('C', C)
S = inner_product_embedding(Dis, dimensions)
# for i in range(R.shape[0]):
# R[i, i] = np.NaN
# C[i, i] = np.NaN
return S
def publish(self, pub):
if self.count < 10:
pub.text('warning', 'Too early to publish anything.')
return
pub.text('info', 'Using statistics: %s' % self.statistic)
if False: # TODO: make option
S = self.get_S_discrete(2, pub=pub.section('computation'))
else:
S = self.get_S(2, pub=pub.section('computation'))
with pub.plot('S') as pylab:
style_ieee_halfcol_xy(pylab)
pylab.plot(S[0, :], S[1, :], 's')
with pub.plot('S_joined') as pylab:
style_ieee_halfcol_xy(pylab)
pylab.plot(S[0, :], S[1, :], '-')
self.y_stats.publish(pub.section('y_stats'))
self.y_dot_stats.publish(pub.section('y_dot_stats'))
self.y_dot_sgn_stats.publish(pub.section('y_dot_sgn_stats'))
self.y_dot_abs_stats.publish(pub.section('y_dot_abs_stats'))
class EstStats(ExpSwitcher):
'''
A simple agent that estimates various statistics
of the observations.
'''
def init(self, boot_spec):
ExpSwitcher.init(self, boot_spec)
if len(boot_spec.get_observations().shape()) != 1:
raise UnsupportedSpec('I assume 1D signals.')
self.y_stats = MeanCovariance()
def merge(self, other):
self.y_stats.merge(other.y_stats)
def process_observations(self, obs):
y = obs['observations']
dt = obs['dt'].item()
self.y_stats.update(y, dt)
def get_state(self):
return dict(y_stats=self.y_stats)
def set_state(self, state):
self.y_stats = state['y_stats']
def publish(self, pub):
if self.y_stats.get_num_samples() == 0:
pub.text('warning', 'Too early to publish anything.')
return
Py = self.y_stats.get_covariance()
Ry = self.y_stats.get_correlation()
Py_inv = self.y_stats.get_information()
Ey = self.y_stats.get_mean()
y_max = self.y_stats.get_maximum()
y_min = self.y_stats.get_minimum()
Ry0 = Ry.copy()
np.fill_diagonal(Ry0, np.NaN)
Py0 = Py.copy()
np.fill_diagonal(Py0, np.NaN)
pub.text('stats', 'Num samples: %s' % self.y_stats.get_num_samples())
with pub.plot('y_bounds') as pylab:
style_ieee_fullcol_xy(pylab)
pylab.plot(Ey, label='E(y)')
pylab.plot(y_max, label='y_max')
pylab.plot(y_min, label='y_min')
pylab.legend()
all_positive = (np.min(Ey) > 0
and np.min(y_max) > 0
and np.min(y_min) > 0)
if all_positive:
with pub.plot('y_stats_log') as pylab:
style_ieee_fullcol_xy(pylab)
pylab.semilogy(Ey, label='E(y)')
pylab.semilogy(y_max, label='y_max')
pylab.semilogy(y_min, label='y_min')
pylab.legend()
pub.array_as_image('Py', Py, caption='cov(y)')
pub.array_as_image('Py0', Py0, caption='cov(y) - no diagonal')
pub.array_as_image('Ry', Ry, caption='corr(y)')
pub.array_as_image('Ry0', Ry0, caption='corr(y) - no diagonal')
pub.array_as_image('Py_inv', Py_inv)
pub.array_as_image('Py_inv_n', cov2corr(Py_inv))
with pub.plot('Py_svd') as pylab: # XXX: use spectrum
style_ieee_fullcol_xy(pylab)
_, s, _ = np.linalg.svd(Py)
s /= s[0]
pylab.semilogy(s, 'bx-')
with pub.subsection('y_stats') as sub:
self.y_stats.publish(sub)
def init(self, boot_spec):
ExpSwitcher.init(self, boot_spec)
if len(boot_spec.get_observations().shape()) != 1:
raise UnsupportedSpec('I assume 1D signals.')
self.y_stats = MeanCovariance()
class Importance(object):
def __init__(self, max_y_dot, max_gy, min_y, max_y):
self.min_y = min_y
self.max_y = max_y
self.max_y_dot = max_y_dot
self.max_gy = max_gy
self.w_stats = MeanCovariance()
warnings.warn('This is not invariant to linear transformation (max,min y).')
self.once = False
@contract(y='array[N]', y_dot='array[N]', returns='array[N]')
def get_importance(self, y, y_dot):
self.once = True
gy = generalized_gradient(y) # gy='array[1xN]',
y_valid = np.logical_and(y > self.min_y, y < self.max_y)
gy0 = gy[0, :]
gy_valid = np.abs(gy0) < self.max_gy
y_dot_valid = np.abs(y_dot) < self.max_y_dot
w = y_valid * 1.0 * gy_valid * y_dot_valid
self.w_stats.update(w)
self.last_w = w
self.last_y = y
self.last_y_valid = y_valid
self.last_y_dot = y_dot
self.last_y_dot_valid = y_dot_valid
self.last_gy = gy
self.last_gy_valid = gy_valid
return w
def publish(self, pub):
if not self.once:
pub.text('info', 'never called yet')
return
N = self.last_y.size
with pub.plot('last_w') as pylab:
pylab.plot(self.last_w, 's')
x_axis_set(pylab, -1, N)
y_axis_set(pylab, -0.1, 1.1)
turn_off_bottom_and_top(pylab)
gy0 = self.last_gy[0, :]
def plot_good_bad(pylab, x, valid):
invalid = np.logical_not(valid)
pylab.plot(np.nonzero(valid)[0], x[valid], 'ks')
pylab.plot(np.nonzero(invalid)[0], x[invalid], 'rs')
with pub.plot('last_y') as pylab:
plot_good_bad(pylab, self.last_y, self.last_y_valid)
y_axis_set(pylab, -0.1, +1.1)
x_axis_set(pylab, -1, N)
turn_off_bottom_and_top(pylab)
with pub.plot('last_y_dot') as pylab:
pylab.plot(self.last_y_dot)
plot_good_bad(pylab, self.last_y_dot, self.last_y_dot_valid)
upper = np.ones(N) * self.max_y_dot
lower = -upper
pylab.plot(upper, 'r--')
pylab.plot(lower, 'r--')
x_axis_set(pylab, -1, N)
y_axis_balanced(pylab)
turn_off_bottom_and_top(pylab)
with pub.plot('last_gy') as pylab:
plot_good_bad(pylab, gy0, self.last_gy_valid)
upper = np.ones(N) * self.max_gy
lower = -upper
pylab.plot(upper, 'r--')
pylab.plot(lower, 'r--')
x_axis_set(pylab, -1, N)
y_axis_balanced(pylab)
turn_off_bottom_and_top(pylab)
self.w_stats.publish(pub.section('w_stats'))
class BDSEEstimator(BDSEEstimatorInterface):
"""
Estimates a BDSE model.
Tensors used: ::
M^s_vi (N) x (N x K)
N^s_i (N) x (K)
T^svi (NxNxK)
U^si (NxK)
"""
@contract(rcond='float,>0')
def __init__(self, rcond=1e-10, antisym_T=False, antisym_M=False, use_P_scaling=False):
"""
:param rcond: Threshold for computing pseudo-inverse of P.
:param antisym_T: If True, the estimate of T is antisymmetrized.
:param antisym_M: If True, the estimate of M is antisymmetrized.
"""
self.rcond = rcond
self.antisym_M = antisym_M
self.antisym_T = antisym_T
self.use_P_scaling = use_P_scaling
self.info('rcond: %f' % rcond)
self.info('antisym_T: %s' % antisym_T)
self.info('antisym_M: %s' % antisym_M)
self.info('use_P_scaling: %s' % use_P_scaling)
self.T = Expectation()
self.U = Expectation()
self.y_stats = MeanCovariance()
self.u_stats = MeanCovariance()
self.nsamples = 0
self.once = False
def merge(self, other):
assert isinstance(other, BDSEEstimator)
self.T.merge(other.T)
self.U.merge(other.U)
self.y_stats.merge(other.y_stats)
self.u_stats.merge(other.u_stats)
self.nsamples += other.nsamples
@contract(u='array[K],K>0,finite',
y='array[N],N>0,finite',
y_dot='array[N],finite', w='>0')
def update(self, y, u, y_dot, w=1.0):
self.once = True
self.nsamples += 1
self.n = y.size
self.k = u.size # XXX: check
self.y_stats.update(y, w)
self.u_stats.update(u, w)
# remove mean
u_n = u - self.u_stats.get_mean()
y_n = y - self.y_stats.get_mean()
# make products
T_k = outer(outer(y_n, y_dot), u_n)
assert T_k.shape == (self.n, self.n, self.k)
U_k = outer(y_dot, u_n)
assert U_k.shape == (self.n, self.k)
# update tensor
self.T.update(T_k, w)
self.U.update(U_k, w)
def get_P_inv_cond(self):
P = self.y_stats.get_covariance()
if False:
P_inv = np.linalg.pinv(P, rcond=self.rcond)
if True:
P2 = P + np.eye(P.shape[0]) * self.rcond
P_inv = np.linalg.inv(P2)
return P_inv
def get_T(self):
T = self.T.get_value()
if self.antisym_T:
self.info('antisymmetrizing T')
T = antisym(T)
return T
def get_model(self):
T = self.get_T()
U = self.U.get_value()
P = self.y_stats.get_covariance()
Q = self.u_stats.get_covariance()
P_inv = self.get_P_inv_cond()
Q_inv = np.linalg.pinv(Q)
if False:
M = get_M_from_P_T_Q(P, T, Q)
else:
if hasattr(self, 'use_P_scaling') and self.use_P_scaling:
M = get_M_from_P_T_Q_alt_scaling(P, T, Q)
else:
warnings.warn('untested')
try:
M = get_M_from_Pinv_T_Q(P_inv, T, Q)
except LinAlgError as e:
msg = 'Could not get_M_from_Pinv_T_Q.\n'
msg += indent(traceback.format_exc(e), '> ')
raise BDSEEstimatorInterface.NotReady(msg)
UQ_inv = np.tensordot(U, Q_inv, axes=(1, 0))
# This works but badly conditioned
Myav = np.tensordot(M, self.y_stats.get_mean(), axes=(1, 0))
N = UQ_inv - Myav
if self.antisym_M:
self.info('antisymmetrizing M')
M = antisym(M)
# # Note: this does not work, don't know why
# if False:
# printm('MYav1', Myav)
# y2 = np.linalg.solve(P, self.y_stats.get_mean())
# Myav2 = np.tensordot(T, y2, axes=(0, 0))
# # Myav = np.tensordot(T, y2, axes=(1, 0))
# printm('MYav2', Myav2)
# if False:
# printm('U', U, 'Q_inv', Q_inv)
# printm('UQ_inv', UQ_inv, 'Myav', Myav, 'N', N)
# printm('u_mean', self.u_stats.get_mean())
# printm('u_std', np.sqrt(Q.diagonal()))
# printm('y_mean', self.y_stats.get_mean())
self.Myav = Myav
self.UQ_inv = UQ_inv
return BDSEmodel(M, N)
def publish(self, pub):
if not self.once:
pub.text('warning', 'not updated yet')
return
pub.text('nsamples', '%s' % self.nsamples)
pub.text('rcond', '%g' % self.rcond)
with pub.subsection('model') as sub:
try:
model = self.get_model()
model.publish(sub)
except BDSEEstimatorInterface.NotReady as e:
pub.text('not-ready', str(e))
with pub.subsection('tensors') as sub:
T = self.get_T()
U = self.U.get_value()
P = self.y_stats.get_covariance()
Q = self.u_stats.get_covariance()
P_inv = np.linalg.pinv(P)
P_inv_cond = self.get_P_inv_cond()
Q_inv = np.linalg.pinv(Q)
#
# TP_inv2 = obtain_TP_inv_from_TP_2(T, P)
# M2 = np.tensordot(TP_inv2, Q_inv, axes=(2, 0))
pub_tensor3_slice2(sub, 'T', T)
pub_tensor2_comp1(sub, 'U', U)
pub_tensor2_cov(sub, 'P', P, rcond=self.rcond)
pub_tensor2_cov(sub, 'P_inv', P_inv)
pub_tensor2_cov(sub, 'P_inv_cond', P_inv_cond)
pub_tensor2_cov(sub, 'Q', Q)
pub_tensor2_cov(sub, 'Q_inv', Q_inv)
# Might not have been computed
# pub_tensor2_comp1(sub, 'Myav', self.Myav)
# pub_tensor2_comp1(sub, 'UQ_inv', self.UQ_inv)
with pub.subsection('y_stats') as sub:
self.y_stats.publish(sub)
with pub.subsection('u_stats') as sub:
self.u_stats.publish(sub)
with pub.subsection('alternative', robust=True) as sub:
sub.text('info', 'This is estimating without conditioning P')
T = self.get_T()
P = self.y_stats.get_covariance()
Q = self.u_stats.get_covariance()
M1 = get_M_from_P_T_Q(P, T, Q)
pub_tensor3_slice2(sub, 'get_M_from_P_T_Q', M1)
M2 = get_M_from_P_T_Q_alt(P, T, Q)
pub_tensor3_slice2(sub, 'get_M_from_P_T_Q_alt', M2)
M3 = get_M_from_P_T_Q_alt_scaling(P, T, Q)
pub_tensor3_slice2(sub, 'get_M_from_P_T_Q_alt2', M3)
class BDSEEstimatorRobust(BDSEEstimator):
def __init__(self, **other):
BDSEEstimator.__init__(self, **other)
self.T = ExpectationWeighted()
self.U = ExpectationWeighted()
self.y_mean = ExpectationWeighted() # XXX: not necessary, y_stats.get_mean()
self.y_stats = MeanCovariance() # TODO: make robust
self.u_stats = MeanCovariance()
self.once = False
def merge(self, other):
assert isinstance(other, BDSEEstimatorRobust)
self.T.merge(other.T)
self.U.merge(other.U)
self.y_stats.merge(other.y_stats)
self.y_mean.merge(other.y_mean)
self.u_stats.merge(other.u_stats)
@contract(u='array[K],K>0,finite',
y='array[N],N>0,finite',
y_dot='array[N],finite',
w='array[N]')
def update(self, y, u, y_dot, w):
self.once = True
self.nsamples += 1
check_all_finite(y)
check_all_finite(u)
check_all_finite(y_dot)
check_all_finite(w)
self.n = y.size
self.k = u.size
self.y_stats.update(y) # TODO: make robust
self.u_stats.update(u)
# remove mean
u_n = u - self.u_stats.get_mean()
self.y_mean.update(y, w) # TODO: make robust
y_n = y - self.y_mean.get_value(fill_value=0.5)
# weights
y_n_w = w
y_dot_w = w
u_n_w = np.ones(u.shape)
T_k = outer(outer(y_n, y_dot), u_n)
T_k_w = outer(outer(y_n_w, y_dot_w), u_n_w)
U_k = outer(y_dot, u_n)
U_k_w = outer(y_dot_w, u_n_w)
assert T_k.shape == (self.n, self.n, self.k)
assert U_k.shape == (self.n, self.k)
# update tensor
self.T.update(T_k, T_k_w)
self.U.update(U_k, U_k_w)
class BDSEAgent(AgentInterface):
'''
An agent that uses a BDS model.
'''
@contract(servo='code_spec', estimator='code_spec')
def __init__(self, explorer, servo, estimator, skip=1,
change_fraction=0.0):
"""
:param explorer: ID of the explorer agent.
:param servo: extra parameters for servo; if string, the ID of an agent.
:param skip: only used one every skip observations.
"""
boot_config = get_boot_config()
_, self.explorer = boot_config.agents.instance_smarter(explorer) # @UndefinedVariable
self.skip = skip
self.change_fraction = change_fraction
self.servo = servo
self.estimator_spec = estimator
def init(self, boot_spec):
self.boot_spec = boot_spec
if len(boot_spec.get_observations().shape()) != 1:
raise UnsupportedSpec('This agent can only work with 1D signals.')
self.count = 0
self.rd = RemoveDoubles(self.change_fraction)
self.y_deriv = DerivativeBox()
self.bdse_estimator = instantiate_spec(self.estimator_spec)
if not isinstance(self.bdse_estimator, BDSEEstimatorInterface):
msg = ('Expected a BDSEEstimatorInterface, got %s'
% describe_type(self.estimator))
raise ValueError(msg)
self.y_stats = MeanCovariance()
self.explorer.init(boot_spec)
self.commands_spec = boot_spec.get_commands()
# All the rest are only statistics
self.stats = MiscStatistics()
def choose_commands(self):
return self.explorer.choose_commands()
def process_observations(self, obs):
self.explorer.process_observations(obs)
self.count += 1
if self.count % self.skip != 0:
return
dt = float(obs['dt'])
y = obs['observations']
u = obs['commands']
# TODO: abstract away
self.rd.update(y)
if not self.rd.ready():
return
# XXX: this is not `dt` anymore FiXME:
self.y_stats.update(y, dt)
if obs['episode_start']:
# self.info('episode_changed: %s' % obs['id_episode'])
self.y_deriv.reset()
return
self.y_deriv.update(y, dt)
if not self.y_deriv.ready():
return
y_sync, y_dot_sync = self.y_deriv.get_value()
self.bdse_estimator.update(u=u.astype('float32'),
y=y_sync.astype('float32'),
y_dot=y_dot_sync.astype('float32'),
w=dt)
# Just other statistics
self.stats.update(y_sync, y_dot_sync, u, dt)
def publish(self, pub):
if self.count < 10:
self.info('Skipping publishing as count=%d' % self.count)
return
with pub.subsection('estimator') as sub:
self.bdse_estimator.publish(sub)
with pub.subsection('stats') as sub:
self.stats.publish(sub)
def get_predictor(self):
model = self.bdse_estimator.get_model()
return BDSEPredictor(model)
def get_servo(self):
servo_agent = instantiate_spec(self.servo)
servo_agent.init(self.boot_spec)
assert isinstance(servo_agent, BDSEServoInterface)
model = self.bdse_estimator.get_model()
servo_agent.set_model(model)
return servo_agent
def merge(self, agent2):
assert isinstance(agent2, BDSEAgent)
self.bdse_estimator.merge(agent2.bdse_estimator)
def __init__(self):
self.y_stats = MeanCovariance()
self.y_dot_stats = MeanCovariance()
self.y_dot_abs_stats = MeanCovariance()
self.u_stats = MeanCovariance()
self.dt_stats = MeanCovariance()
class MiscStatistics(object):
def __init__(self):
self.y_stats = MeanCovariance()
self.y_dot_stats = MeanCovariance()
self.y_dot_abs_stats = MeanCovariance()
self.u_stats = MeanCovariance()
self.dt_stats = MeanCovariance()
def update(self, y, y_dot, u, dt):
self.y_stats.update(y, dt)
self.dt_stats.update(np.array([dt]))
self.u_stats.update(u, dt)
self.y_dot_stats.update(y_dot, dt)
self.y_dot_abs_stats.update(np.abs(y_dot), dt)
def publish(self, pub):
self.y_stats.publish(pub.section('y_stats'))
self.u_stats.publish(pub.section('u_stats'))
self.y_dot_stats.publish(pub.section('y_dot_stats'))
self.y_dot_abs_stats.publish(pub.section('y_dot_abs_stats'))
self.dt_stats.publish(pub.section('dt_stats'))
