def _execute(self, x):
#----------------------------------------------------
# similar algorithm to that within self.stop_training()
# refer there for notes & comments on code
#----------------------------------------------------
N = self.data.shape[0]
Nx = x.shape[0]
W = numx.zeros((Nx, N), dtype=self.dtype)
k, r = self.k, self.r
d_out = self.output_dim
Q_diag_idx = numx.arange(k)
for row in range(Nx):
#find nearest neighbors of x in M
M_xi = self.data - x[row]
nbrs = numx.argsort((M_xi**2).sum(1))[:k]
M_xi = M_xi[nbrs]
#find corrected covariance matrix Q
Q = mult(M_xi, M_xi.T)
if r is None and k > d_out:
sig2 = (svd(M_xi, compute_uv=0))**2
r = numx.sum(sig2[d_out:])
Q[Q_diag_idx, Q_diag_idx] += r
if r is not None:
Q[Q_diag_idx, Q_diag_idx] += r
#solve for weights
w = self._refcast(numx_linalg.solve(Q, numx.ones(k)))
w /= w.sum()
W[row, nbrs] = w
#multiply weights by result of SVD from training
return numx.dot(W, self.training_projection)
def _execute(self, x):
#----------------------------------------------------
# similar algorithm to that within self.stop_training()
# refer there for notes & comments on code
#----------------------------------------------------
N = self.data.shape[0]
Nx = x.shape[0]
W = numx.zeros((Nx, N), dtype=self.dtype)
k, r = self.k, self.r
d_out = self.output_dim
Q_diag_idx = numx.arange(k)
for row in range(Nx):
#find nearest neighbors of x in M
M_xi = self.data-x[row]
nbrs = numx.argsort( (M_xi**2).sum(1) )[:k]
M_xi = M_xi[nbrs]
#find corrected covariance matrix Q
Q = mult(M_xi, M_xi.T)
if r is None and k > d_out:
sig2 = (svd(M_xi, compute_uv=0))**2
r = numx.sum(sig2[d_out:])
Q[Q_diag_idx, Q_diag_idx] += r
if r is not None:
Q[Q_diag_idx, Q_diag_idx] += r
#solve for weights
w = self._refcast(numx_linalg.solve(Q , numx.ones(k)))
w /= w.sum()
W[row, nbrs] = w
#multiply weights by result of SVD from training
return numx.dot(W, self.training_projection)
def _update_mean(self, x, label):
"""Update the mean with data for a single label."""
if label not in self.label_means:
self.label_means[label] = numx.zeros(self.input_dim)
self.n_label_samples[label] = 0
# TODO: use smarter summing to avoid rounding errors
self.label_means[label] += numx.sum(x, axis=0)
self.n_label_samples[label] += len(x)
def _update_mean(self, x, label):
"""Update the mean with data for a single label."""
if label not in self.label_means:
self.label_means[label] = numx.zeros(self.input_dim)
self.n_label_samples[label] = 0
# TODO: use smarter summing to avoid rounding errors
self.label_means[label] += numx.sum(x, axis=0)
self.n_label_samples[label] += len(x)
def _update_mean(self, x, label):
"""Update the mean with data for a single label.
:param x: The data.
:type x: numpy.ndarray
:param label: The label index.
"""
if label not in self.label_means:
self.label_means[label] = numx.zeros(self.input_dim)
self.n_label_samples[label] = 0
# TODO: use smarter summing to avoid rounding errors
self.label_means[label] += numx.sum(x, axis=0)
self.n_label_samples[label] += len(x)
def _inverse(self, x):
"""Take the mean of overlapping values."""
n_y_cons = numx.bincount(self.connections) # n. connections to y_i
y_cons = numx.argsort(self.connections) # x indices for y_i
y = numx.zeros((len(x), self.input_dim))
i_x_counter = 0 # counter for processed x indices
i_y = 0 # current y index
while True:
n_cons = n_y_cons[i_y]
if n_cons > 0:
y[:, i_y] = old_div(
numx.sum(x[:, y_cons[i_x_counter:i_x_counter + n_cons]],
axis=1), n_cons)
i_x_counter += n_cons
if i_x_counter >= self.output_dim:
break
i_y += 1
return y
def _inverse(self, x):
"""Take the mean of overlapping values."""
n_y_cons = numx.bincount(self.connections) # n. connections to y_i
y_cons = numx.argsort(self.connections) # x indices for y_i
y = numx.zeros((len(x), self.input_dim))
i_x_counter = 0 # counter for processed x indices
i_y = 0 # current y index
while True:
n_cons = n_y_cons[i_y]
if n_cons > 0:
y[:,i_y] = old_div(numx.sum(x[:,y_cons[i_x_counter:
i_x_counter + n_cons]],
axis=1), n_cons)
i_x_counter += n_cons
if i_x_counter >= self.output_dim:
break
i_y += 1
return y
def test_gradient_product(self):
"""Test that the product of gradients is calculated correctly."""
sfa_node1 = bimdp.nodes.SFABiNode(output_dim=5)
sfa_node2 = bimdp.nodes.SFABiNode(output_dim=3)
flow = sfa_node1 + sfa_node2
x = numx_rand.random((300, 10))
flow.train(x)
mdp.activate_extension("gradient")
try:
x1 = numx_rand.random((2, 10))
x2, msg = sfa_node1.execute(x1, {"method": "gradient"})
grad1 = msg["grad"]
_, msg = sfa_node2.execute(x2, {"method": "gradient"})
grad2 = msg["grad"]
grad12 = flow.execute(x1, {"method": "gradient"})[1]["grad"]
# use a different way to calculate the product of the gradients,
# this method is too memory intensive for large data
ref_grad = numx.sum(grad2[:,:,numx.newaxis,:] *
numx.transpose(grad1[:,numx.newaxis,:,:], (0,1,3,2)),
axis=3)
assert numx.amax(abs(ref_grad - grad12)) < 1E-9
finally:
mdp.deactivate_extension("gradient")
def test_gradient_product(self):
"""Test that the product of gradients is calculated correctly."""
sfa_node1 = bimdp.nodes.SFABiNode(output_dim=5)
sfa_node2 = bimdp.nodes.SFABiNode(output_dim=3)
flow = sfa_node1 + sfa_node2
x = numx_rand.random((300, 10))
flow.train(x)
mdp.activate_extension("gradient")
try:
x1 = numx_rand.random((2, 10))
x2, msg = sfa_node1.execute(x1, {"method": "gradient"})
grad1 = msg["grad"]
_, msg = sfa_node2.execute(x2, {"method": "gradient"})
grad2 = msg["grad"]
grad12 = flow.execute(x1, {"method": "gradient"})[1]["grad"]
# use a different way to calculate the product of the gradients,
# this method is too memory intensive for large data
ref_grad = numx.sum(grad2[:, :, numx.newaxis, :] *
numx.transpose(grad1[:, numx.newaxis, :, :],
(0, 1, 3, 2)),
axis=3)
assert numx.amax(abs(ref_grad - grad12)) < 1E-9
finally:
mdp.deactivate_extension("gradient")
def symeig_semidefinite_reg(A,
B=None,
eigenvectors=True,
turbo="on",
range=None,
type=1,
overwrite=False,
rank_threshold=1e-12,
dfc_out=None):
"""Regularization-based routine to solve generalized symmetric positive
semidefinite eigenvalue problems.
This can be used if the normal ``symeig()`` call in ``_stop_training()``
throws ``SymeigException('Covariance matrices may be singular')``.
This solver applies a moderate regularization to B before applying
``eigh``/``symeig``. Afterwards it properly detects the rank deficit and
filters out malformed features.
For full range, this procedure is (approximately) as efficient as the
ordinary ``eigh`` implementation, because all additional steps are
computationally cheap.
For shorter range, the LDL method should be preferred.
.. note::
For efficiency reasons it actually modifies the matrix B
(even if ``overwrite=False``), but the changes are negligible.
The signature of this function equals that of ``mdp.utils.symeig``,
but has two additional parameters:
:param rank_threshold:
A threshold to determine if an eigenvalue counts as zero.
:type rank_threshold: float
:param dfc_out:
If ``dfc_out`` is not ``None``, ``dfc_out.rank_deficit`` will be set
to an integer indicating how many zero-eigenvalues were detected.
"""
if type != 1:
raise ValueError('Only type=1 is supported.')
# apply some regularization...
# The following is equivalent to B += 1e-12*np.eye(B.shape[0]),
# but works more in place, i.e. saves memory consumption of np.eye().
Bflat = B.reshape(B.shape[0] * B.shape[1])
idx = numx.arange(0, len(Bflat), B.shape[0] + 1)
diag_tmp = Bflat[idx]
Bflat[idx] += rank_threshold
eg, ev = mdp.utils.symeig(A, B, True, turbo, None, type, overwrite)
Bflat[idx] = diag_tmp
m = numx.absolute(
numx.sqrt(numx.absolute(numx.sum(ev * mdp.utils.mult(B, ev), 0))) - 1)
off = 0
# In theory all values in m should be close to one or close to zero.
# So we use the mean of these values as threshold to distinguish cases:
while m[off] > 0.5:
off += 1
m_off_sum = numx.sum(m[off:])
if m_off_sum < 0.5:
if off > 0:
if not dfc_out is None:
dfc_out.rank_deficit = off
eg = eg[off:]
ev = ev[:, off:]
else:
# Sometimes (unlikely though) the values in m are not sorted
# In this case we search all indices:
m_idx = (m < 0.5).nonzero()[0]
eg = eg[m_idx]
ev = ev[:, m_idx]
if range is None:
return eg, ev
else:
return eg[range[0] - 1:range[1]], ev[:, range[0] - 1:range[1]]
def _stop_training(self):
Cumulator._stop_training(self)
if self.verbose:
msg = ('training LLE on %i points'
' in %i dimensions...' %
(self.data.shape[0], self.data.shape[1]))
print msg
# some useful quantities
M = self.data
N = M.shape[0]
k = self.k
r = self.r
# indices of diagonal elements
W_diag_idx = numx.arange(N)
Q_diag_idx = numx.arange(k)
if k > N:
err = ('k=%i must be less than or '
'equal to number of training points N=%i' % (k, N))
raise TrainingException(err)
# determines number of output dimensions: if desired_variance
# is specified, we need to learn it from the data. Otherwise,
# it's easy
learn_outdim = False
if self.output_dim is None:
if self.desired_variance is None:
self.output_dim = self.input_dim
else:
learn_outdim = True
# do we need to automatically determine the regularization term?
auto_reg = r is None
# determine number of output dims, precalculate useful stuff
if learn_outdim:
Qs, sig2s, nbrss = self._adjust_output_dim()
# build the weight matrix
#XXX future work:
#XXX for faster implementation, W should be a sparse matrix
W = numx.zeros((N, N), dtype=self.dtype)
if self.verbose:
print ' - constructing [%i x %i] weight matrix...' % W.shape
for row in range(N):
if learn_outdim:
Q = Qs[row, :, :]
nbrs = nbrss[row, :]
else:
# -----------------------------------------------
# find k nearest neighbors
# -----------------------------------------------
M_Mi = M - M[row]
nbrs = numx.argsort((M_Mi**2).sum(1))[1:k + 1]
M_Mi = M_Mi[nbrs]
# compute covariance matrix of distances
Q = mult(M_Mi, M_Mi.T)
# -----------------------------------------------
# compute weight vector based on neighbors
# -----------------------------------------------
#Covariance matrix may be nearly singular:
# add a diagonal correction to prevent numerical errors
if auto_reg:
# automatic mode: correction is equal to the sum of
# the (d_in-d_out) unused variances (as in deRidder &
# Duin)
if learn_outdim:
sig2 = sig2s[row, :]
else:
sig2 = svd(M_Mi, compute_uv=0)**2
r = numx.sum(sig2[self.output_dim:])
Q[Q_diag_idx, Q_diag_idx] += r
else:
# Roweis et al instead use "a correction that
# is small compared to the trace" e.g.:
# r = 0.001 * float(Q.trace())
# this is equivalent to assuming 0.1% of the variance is unused
Q[Q_diag_idx, Q_diag_idx] += r * Q.trace()
#solve for weight
# weight is w such that sum(Q_ij * w_j) = 1 for all i
# XXX refcast is due to numpy bug: floats become double
w = self._refcast(numx_linalg.solve(Q, numx.ones(k)))
w /= w.sum()
#update row of the weight matrix
W[nbrs, row] = w
if self.verbose:
msg = (' - finding [%i x %i] null space of weight matrix\n'
' (may take a while)...' % (self.output_dim, N))
print msg
self.W = W.copy()
#to find the null space, we need the bottom d+1
# eigenvectors of (W-I).T*(W-I)
#Compute this using the svd of (W-I):
W[W_diag_idx, W_diag_idx] -= 1.
#XXX future work:
#XXX use of upcoming ARPACK interface for bottom few eigenvectors
#XXX of a sparse matrix will significantly increase the speed
#XXX of the next step
if self.svd:
sig, U = nongeneral_svd(W.T, range=(2, self.output_dim + 1))
else:
# the following code does the same computation, but uses
# symeig, which computes only the required eigenvectors, and
# is much faster. However, it could also be more unstable...
WW = mult(W, W.T)
# regularizes the eigenvalues, does not change the eigenvectors:
WW[W_diag_idx, W_diag_idx] += 0.1
sig, U = symeig(WW, range=(2, self.output_dim + 1), overwrite=True)
self.training_projection = U
def symeig_semidefinite_reg(
A, B = None, eigenvectors=True, turbo="on", range=None,
type=1, overwrite=False, rank_threshold=1e-12, dfc_out=None):
"""
Regularization-based routine to solve generalized symmetric positive
semidefinite eigenvalue problems.
This can be used in case the normal symeig() call in _stop_training()
throws SymeigException ('Covariance matrices may be singular').
This solver applies a moderate regularization to B before applying
eigh/symeig. Afterwards it properly detects the rank deficit and
filters out malformed features.
For full range, this procedure is (approximately) as efficient as the
ordinary eigh implementation, because all additional steps are
computationally cheap.
For shorter range, the LDL method should be preferred.
The signature of this function equals that of mdp.utils.symeig, but
has two additional parameters:
rank_threshold: A threshold to determine if an eigenvalue counts as zero.
dfc_out: If dfc_out is not None dfc_out.rank_deficit will be set to an
integer indicating how many zero-eigenvalues were detected.
Note:
For efficiency reasons it actually modifies the matrix B
(even if overwrite=False), but the changes are negligible.
"""
if type != 1:
raise ValueError('Only type=1 is supported.')
# apply some regularization...
# The following is equivalent to B += 1e-12*np.eye(B.shape[0]),
# but works more in place, i.e. saves memory consumption of np.eye().
Bflat = B.reshape(B.shape[0]*B.shape[1])
idx = numx.arange(0, len(Bflat), B.shape[0]+1)
diag_tmp = Bflat[idx]
Bflat[idx] += rank_threshold
eg, ev = mdp.utils.symeig(A, B, True, turbo, None, type, overwrite)
Bflat[idx] = diag_tmp
m = numx.absolute(numx.sqrt(numx.absolute(
numx.sum(ev * mdp.utils.mult(B, ev), 0)))-1)
off = 0
# In theory all values in m should be close to one or close to zero.
# So we use the mean of these values as threshold to distinguish cases:
while m[off] > 0.5:
off += 1
m_off_sum = numx.sum(m[off:])
if m_off_sum < 0.5:
if off > 0:
if not dfc_out is None:
dfc_out.rank_deficit = off
eg = eg[off:]
ev = ev[:, off:]
else:
# Sometimes (unlikely though) the values in m are not sorted
# In this case we search all indices:
m_idx = (m < 0.5).nonzero()[0]
eg = eg[m_idx]
ev = ev[:, m_idx]
if range is None:
return eg, ev
else:
return eg[range[0]-1:range[1]], ev[:, range[0]-1:range[1]]
def _stop_training(self):
Cumulator._stop_training(self)
if self.verbose:
msg = ('training LLE on %i points'
' in %i dimensions...' % (self.data.shape[0],
self.data.shape[1]))
print msg
# some useful quantities
M = self.data
N = M.shape[0]
k = self.k
r = self.r
# indices of diagonal elements
W_diag_idx = numx.arange(N)
Q_diag_idx = numx.arange(k)
if k > N:
err = ('k=%i must be less than or '
'equal to number of training points N=%i' % (k, N))
raise TrainingException(err)
# determines number of output dimensions: if desired_variance
# is specified, we need to learn it from the data. Otherwise,
# it's easy
learn_outdim = False
if self.output_dim is None:
if self.desired_variance is None:
self.output_dim = self.input_dim
else:
learn_outdim = True
# do we need to automatically determine the regularization term?
auto_reg = r is None
# determine number of output dims, precalculate useful stuff
if learn_outdim:
Qs, sig2s, nbrss = self._adjust_output_dim()
# build the weight matrix
#XXX future work:
#XXX for faster implementation, W should be a sparse matrix
W = numx.zeros((N, N), dtype=self.dtype)
if self.verbose:
print ' - constructing [%i x %i] weight matrix...' % W.shape
for row in range(N):
if learn_outdim:
Q = Qs[row, :, :]
nbrs = nbrss[row, :]
else:
# -----------------------------------------------
# find k nearest neighbors
# -----------------------------------------------
M_Mi = M-M[row]
nbrs = numx.argsort((M_Mi**2).sum(1))[1:k+1]
M_Mi = M_Mi[nbrs]
# compute covariance matrix of distances
Q = mult(M_Mi, M_Mi.T)
# -----------------------------------------------
# compute weight vector based on neighbors
# -----------------------------------------------
#Covariance matrix may be nearly singular:
# add a diagonal correction to prevent numerical errors
if auto_reg:
# automatic mode: correction is equal to the sum of
# the (d_in-d_out) unused variances (as in deRidder &
# Duin)
if learn_outdim:
sig2 = sig2s[row, :]
else:
sig2 = svd(M_Mi, compute_uv=0)**2
r = numx.sum(sig2[self.output_dim:])
Q[Q_diag_idx, Q_diag_idx] += r
else:
# Roweis et al instead use "a correction that
# is small compared to the trace" e.g.:
# r = 0.001 * float(Q.trace())
# this is equivalent to assuming 0.1% of the variance is unused
Q[Q_diag_idx, Q_diag_idx] += r*Q.trace()
#solve for weight
# weight is w such that sum(Q_ij * w_j) = 1 for all i
# XXX refcast is due to numpy bug: floats become double
w = self._refcast(numx_linalg.solve(Q, numx.ones(k)))
w /= w.sum()
#update row of the weight matrix
W[nbrs, row] = w
if self.verbose:
msg = (' - finding [%i x %i] null space of weight matrix\n'
' (may take a while)...' % (self.output_dim, N))
print msg
self.W = W.copy()
#to find the null space, we need the bottom d+1
# eigenvectors of (W-I).T*(W-I)
#Compute this using the svd of (W-I):
W[W_diag_idx, W_diag_idx] -= 1.
#XXX future work:
#XXX use of upcoming ARPACK interface for bottom few eigenvectors
#XXX of a sparse matrix will significantly increase the speed
#XXX of the next step
if self.svd:
sig, U = nongeneral_svd(W.T, range=(2, self.output_dim+1))
else:
# the following code does the same computation, but uses
# symeig, which computes only the required eigenvectors, and
# is much faster. However, it could also be more unstable...
WW = mult(W, W.T)
# regularizes the eigenvalues, does not change the eigenvectors:
WW[W_diag_idx, W_diag_idx] += 0.1
sig, U = symeig(WW, range=(2, self.output_dim+1), overwrite=True)
self.training_projection = U
