def get_eta_values(self, t=1):
"""Return the eta values of the slow components learned during
the training phase. If the training phase has not been completed
yet, call `stop_training`.
The delta value of a signal is a measure of its temporal
variation, and is defined as the mean of the derivative squared,
i.e. delta(x) = mean(dx/dt(t)^2). delta(x) is zero if
x is a constant signal, and increases if the temporal variation
of the signal is bigger.
The eta value is a more intuitive measure of temporal variation,
defined as
eta(x) = t/(2*pi) * sqrt(delta(x))
If x is a signal of length 't' which consists of a sine function
that accomplishes exactly N oscillations, then eta(x)=N.
:param t: Sampling frequency in Hz.
The original definition in (Wiskott and Sejnowski, 2002)
is obtained for t = number of training data points, while
for t=1 (default), this corresponds to the beta-value defined
in (Berkes and Wiskott, 2005).
:returns: The eta values of the slow components learned during
the training phase.
"""
if self.is_training():
self.stop_training()
return self._refcast(t / (2 * numx.pi) * numx.sqrt(self.d))
def get_eta_values(self, t=1):
"""Return the eta values of the slow components learned during
the training phase. If the training phase has not been completed
yet, call `stop_training`.
The delta value of a signal is a measure of its temporal
variation, and is defined as the mean of the derivative squared,
i.e. delta(x) = mean(dx/dt(t)^2). delta(x) is zero if
x is a constant signal, and increases if the temporal variation
of the signal is bigger.
The eta value is a more intuitive measure of temporal variation,
defined as
eta(x) = t/(2*pi) * sqrt(delta(x))
If x is a signal of length 't' which consists of a sine function
that accomplishes exactly N oscillations, then eta(x)=N.
:Parameters:
t
Sampling frequency in Hz.
The original definition in (Wiskott and Sejnowski, 2002)
is obtained for t = number of training data points, while
for t=1 (default), this corresponds to the beta-value defined in
(Berkes and Wiskott, 2005).
"""
if self.is_training():
self.stop_training()
return self._refcast(t / (2 * numx.pi) * numx.sqrt(self.d))
def get_eigenvectors(self):
"""Return the eigenvectors of the covariance matrix.
:return: The eigenvectors of the covariance matrix.
:rtype: numpy.ndarray
"""
self._if_training_stop_training()
return numx.sqrt(self.d) * self.v
def get_eigenvectors(self):
"""Return the eigenvectors of the covariance matrix.
:return: The eigenvectors of the covariance matrix.
:rtype: numpy.ndarray
"""
self._if_training_stop_training()
return numx.sqrt(self.d)*self.v
def _symeig_fake(A, B = None, eigenvectors = True, turbo = "on", range = None,
type = 1, overwrite = False):
"""Solve standard and generalized eigenvalue problem for symmetric
(hermitian) definite positive matrices.
This function is a wrapper of LinearAlgebra.eigenvectors or
numarray.linear_algebra.eigenvectors with an interface compatible with symeig.
Syntax:
w,Z = symeig(A)
w = symeig(A,eigenvectors=0)
w,Z = symeig(A,range=(lo,hi))
w,Z = symeig(A,B,range=(lo,hi))
Inputs:
A -- An N x N matrix.
B -- An N x N matrix.
eigenvectors -- if set return eigenvalues and eigenvectors, otherwise
only eigenvalues
turbo -- not implemented
range -- the tuple (lo,hi) represent the indexes of the smallest and
largest (in ascending order) eigenvalues to be returned.
1 <= lo < hi <= N
if range = None, returns all eigenvalues and eigenvectors.
type -- not implemented, always solve A*x = (lambda)*B*x
overwrite -- not implemented
Outputs:
w -- (selected) eigenvalues in ascending order.
Z -- if range = None, Z contains the matrix of eigenvectors,
normalized as follows:
Z^H * A * Z = lambda and Z^H * B * Z = I
where ^H means conjugate transpose.
if range, an N x M matrix containing the orthonormal
eigenvectors of the matrix A corresponding to the selected
eigenvalues, with the i-th column of Z holding the eigenvector
associated with w[i]. The eigenvectors are normalized as above.
"""
dtype = numx.dtype(_greatest_common_dtype([A, B]))
try:
if B is None:
w, Z = numx_linalg.eigh(A)
else:
# make B the identity matrix
wB, ZB = numx_linalg.eigh(B)
_assert_eigenvalues_real_and_positive(wB, dtype)
ZB = ZB.real / numx.sqrt(wB.real)
# transform A in the new basis: A = ZB^T * A * ZB
A = mdp.utils.mult(mdp.utils.mult(ZB.T, A), ZB)
# diagonalize A
w, ZA = numx_linalg.eigh(A)
Z = mdp.utils.mult(ZB, ZA)
except numx_linalg.LinAlgError, exception:
raise SymeigException(str(exception))
def _train(self, input):
g = self.graph
d = self.d
if len(g.nodes)==0:
# if missing, generate two initial nodes at random
# assuming that the input data has zero mean and unit variance,
# choose the random position according to a gaussian distribution
# with zero mean and unit variance
normal = numx_rand.normal
self._add_node(self._refcast(normal(0.0, 1.0, self.input_dim)))
self._add_node(self._refcast(normal(0.0, 1.0, self.input_dim)))
# loop on single data points
for x in input:
self.tlen += 1
# step 2 - find the nearest nodes
# dists are the squared distances of x from n0, n1
(n0, n1), dists = self._get_nearest_nodes(x)
# step 3 - increase age of the emanating edges
for e in n0.get_edges():
e.data.inc_age()
# step 4 - update error
n0.data.cum_error += numx.sqrt(dists[0])
# step 5 - move nearest node and neighbours
self._move_node(n0, x, self.eps_b)
# neighbors undirected
neighbors = n0.neighbors()
for n in neighbors:
self._move_node(n, x, self.eps_n)
# step 6 - update n0<->n1 edge
if n1 in neighbors:
# should be one edge only
edges = n0.get_edges(neighbor=n1)
edges[0].data.age = 0
else:
self._add_edge(n0, n1)
# step 7 - remove old edges
self._remove_old_edges(n0.get_edges())
# step 8 - add a new node each lambda steps
if not self.tlen % self.lambda_ and len(g.nodes) < self.max_nodes:
self._insert_new_node()
# step 9 - decrease errors
for node in g.nodes:
node.data.cum_error *= d
def _train(self, input):
g = self.graph
d = self.d
if len(g.nodes) == 0:
# if missing, generate two initial nodes at random
# assuming that the input data has zero mean and unit variance,
# choose the random position according to a gaussian distribution
# with zero mean and unit variance
normal = numx_rand.normal
self._add_node(self._refcast(normal(0.0, 1.0, self.input_dim)))
self._add_node(self._refcast(normal(0.0, 1.0, self.input_dim)))
# loop on single data points
for x in input:
self.tlen += 1
# step 2 - find the nearest nodes
# dists are the squared distances of x from n0, n1
(n0, n1), dists = self._get_nearest_nodes(x)
# step 3 - increase age of the emanating edges
for e in n0.get_edges():
e.data.inc_age()
# step 4 - update error
n0.data.cum_error += numx.sqrt(dists[0])
# step 5 - move nearest node and neighbours
self._move_node(n0, x, self.eps_b)
# neighbors undirected
neighbors = n0.neighbors()
for n in neighbors:
self._move_node(n, x, self.eps_n)
# step 6 - update n0<->n1 edge
if n1 in neighbors:
# should be one edge only
edges = n0.get_edges(neighbor=n1)
edges[0].data.age = 0
else:
self._add_edge(n0, n1)
# step 7 - remove old edges
self._remove_old_edges(n0.get_edges())
# step 8 - add a new node each lambda steps
if not self.tlen % self.lambda_ and len(g.nodes) < self.max_nodes:
self._insert_new_node()
# step 9 - decrease errors
for node in g.nodes:
node.data.cum_error *= d
def _stop_training(self, debug=False):
"""Stop the training phase.
:param debug: Determines if singular matrices itself are stored in
self.cov_mtx and self.dcov_mtx to be examined, given that
stop_training fails because of singular covmatrices.
Default is False.
:type debug: bool
"""
super(WhiteningNode, self)._stop_training(debug)
##### whiten the filters
# self.v is now the _whitening_ matrix
self.v = old_div(self.v, numx.sqrt(self.d))
def _stop_training(self):
var_tlen = self._tlen-1
# unbiased
var = (self._var - self._mean*self._mean/self._tlen)/var_tlen
# biased
#var = (self._var - self._mean*self._mean/self._tlen)/self._tlen
# old formula: wrong! is neither biased nor unbiased
#var = (self._var/var_tlen) - (self._mean/self._tlen)**2
self._var = var
delta = (self._diff2/self._tlen)/var
self._delta = delta
self._eta = numx.sqrt(delta)/(2*numx.pi)
def nearest_neighbor(self, input):
"""Assign each point in the input data to the nearest node in
the graph. Return the list of the nearest node instances, and
the list of distances.
Executing this function will close the training phase if
necessary."""
super(GrowingNeuralGasNode, self).execute(input)
nodes = []
dists = []
for x in input:
(n0, _), dist = self._get_nearest_nodes(x)
nodes.append(n0)
dists.append(numx.sqrt(dist[0]))
return nodes, dists
def nearest_neighbor(self, input):
"""Assign each point in the input data to the nearest node in
the graph. Return the list of the nearest node instances, and
the list of distances.
Executing this function will close the training phase if
necessary."""
super(GrowingNeuralGasNode, self).execute(input)
nodes = []
dists = []
for x in input:
(n0, n1), dist = self._get_nearest_nodes(x)
nodes.append(n0)
dists.append(numx.sqrt(dist[0]))
return nodes, dists
def _stop_training(self, debug=False):
"""Stop the training phase.
:param debug: Determines if singular matrices itself are stored in
self.cov_mtx and self.dcov_mtx to be examined, given that
stop_training fails because of singular covmatrices.
Default is False.
:type debug: bool
"""
super(WhiteningNode, self)._stop_training(debug)
##### whiten the filters
# self.v is now the _whitening_ matrix
self.v = old_div(self.v, numx.sqrt(self.d))
def _stop_training(self):
var_tlen = self._tlen-1
# unbiased
var = (self._var - self._mean*self._mean/self._tlen)/var_tlen
# biased
#var = (self._var - self._mean*self._mean/self._tlen)/self._tlen
# old formula: wrong! is neither biased nor unbiased
#var = (self._var/var_tlen) - (self._mean/self._tlen)**2
self._var = var
delta = (self._diff2/self._tlen)/var
self._delta = delta
self._eta = numx.sqrt(delta)/(2*numx.pi)
def _stop_training(self):
var_tlen = self._tlen-1
# unbiased
var = old_div((self._var - self._mean*self._mean/self._tlen),var_tlen)
# biased
#var = (self._var - self._mean*self._mean/self._tlen)/self._tlen
# old formula: wrong! is neither biased nor unbiased
#var = (self._var/var_tlen) - (self._mean/self._tlen)**2
self._var = var
delta = old_div((old_div(self._diff2,self._tlen)),var)
self._delta = delta
self._eta = old_div(numx.sqrt(delta),(2*numx.pi))
def _stop_training(self):
var_tlen = self._tlen - 1
# unbiased
var = old_div((self._var - self._mean * self._mean / self._tlen),
var_tlen)
# biased
#var = (self._var - self._mean*self._mean/self._tlen)/self._tlen
# old formula: wrong! is neither biased nor unbiased
#var = (self._var/var_tlen) - (self._mean/self._tlen)**2
self._var = var
delta = old_div((old_div(self._diff2, self._tlen)), var)
self._delta = delta
self._eta = old_div(numx.sqrt(delta), (2 * numx.pi))
def _stop_training(self):
self.labels = self._cov_objs.keys()
self.labels.sort()
nitems = 0
for lbl in self.labels:
cov, mean, p = self._cov_objs[lbl].fix()
nitems += p
self._sqrt_def_covs.append(numx.sqrt(numx_linalg.det(cov)))
if self._sqrt_def_covs[-1] == 0.0:
err = "The covariance matrix is singular for at least " "one class."
raise mdp.NodeException(err)
self.means.append(mean)
self.p.append(p)
self.inv_covs.append(utils.inv(cov))
for i in range(len(self.p)):
self.p[i] /= float(nitems)
del self._cov_objs
def _stop_training(self):
self.labels = self._cov_objs.keys()
self.labels.sort()
nitems = 0
for lbl in self.labels:
cov, mean, p = self._cov_objs[lbl].fix()
nitems += p
self._sqrt_def_covs.append(numx.sqrt(numx_linalg.det(cov)))
if self._sqrt_def_covs[-1] == 0.0:
err = ("The covariance matrix is singular for at least "
"one class.")
raise mdp.NodeException(err)
self.means.append(mean)
self.p.append(p)
self.inv_covs.append(utils.inv(cov))
for i in range(len(self.p)):
self.p[i] /= float(nitems)
del self._cov_objs
def generate_input(self, len_or_y=1, noise=False):
"""Generate data from the prior distribution.
If the training phase has not been completed yet, call stop_training.
:param len_or_y: If integer, it specified the number of observation
to generate. If array, it is used as a set of samples
of the latent variables
:param noise: If true, generation includes the estimated noise
:type noise: bool
:return: The generated data.
:rtype: numpy.ndarray
"""
self._if_training_stop_training()
# set the output dimension if necessary
if self.output_dim is None:
# if the input_dim is not defined, raise an exception
if self.input_dim is None:
errstr = ("Number of input dimensions undefined. Inversion "
"not possible.")
raise NodeException(errstr)
self.output_dim = self.input_dim
if isinstance(len_or_y, int):
size = (len_or_y, self.output_dim)
y = self._refcast(mdp.numx_rand.normal(size=size))
else:
y = self._refcast(len_or_y)
self._check_output(y)
res = mult(y, self.A.T)+self.mu
if noise:
ns = mdp.numx_rand.normal(size=(y.shape[0], self.input_dim))
ns *= numx.sqrt(self.sigma)
res += self._refcast(ns)
return res
def generate_input(self, len_or_y=1, noise=False):
"""Generate data from the prior distribution.
If the training phase has not been completed yet, call stop_training.
:param len_or_y: If integer, it specified the number of observation
to generate. If array, it is used as a set of samples
of the latent variables
:param noise: If true, generation includes the estimated noise
:type noise: bool
:return: The generated data.
:rtype: numpy.ndarray
"""
self._if_training_stop_training()
# set the output dimension if necessary
if self.output_dim is None:
# if the input_dim is not defined, raise an exception
if self.input_dim is None:
errstr = ("Number of input dimensions undefined. Inversion "
"not possible.")
raise NodeException(errstr)
self.output_dim = self.input_dim
if isinstance(len_or_y, int):
size = (len_or_y, self.output_dim)
y = self._refcast(mdp.numx_rand.normal(size=size))
else:
y = self._refcast(len_or_y)
self._check_output(y)
res = mult(y, self.A.T) + self.mu
if noise:
ns = mdp.numx_rand.normal(size=(y.shape[0], self.input_dim))
ns *= numx.sqrt(self.sigma)
res += self._refcast(ns)
return res
def nearest_neighbor(self, input):
"""Assign each point in the input data to the nearest node in
the graph. Return the list of the nearest node instances, and
the list of distances.
Executing this function will close the training phase if
necessary.
:param input: Points to find the nearest node in the graph to.
:type input: numpy.ndarray
:return: The list of the nearest node instances and
the list of distances.
:rtype: tuple
"""
super(GrowingNeuralGasNode, self).execute(input)
nodes = []
dists = []
for x in input:
(n0, _), dist = self._get_nearest_nodes(x)
nodes.append(n0)
dists.append(numx.sqrt(dist[0]))
return nodes, dists
def get_eigenvectors(self):
"""Return the eigenvectors of the covariance matrix."""
self._if_training_stop_training()
return numx.sqrt(self.d)*self.v
def _stop_training(self, debug=False):
super(WhiteningNode, self)._stop_training(debug)
##### whiten the filters
# self.v is now the _whitening_ matrix
self.v = old_div(self.v, numx.sqrt(self.d))
def get_eigenvectors(self):
"""Return the eigenvectors of the covariance matrix."""
self._if_training_stop_training()
return numx.sqrt(self.d)*self.v
def _stop_training(self, debug=False):
super(WhiteningNode, self)._stop_training(debug)
##### whiten the filters
# self.v is now the _whitening_ matrix
self.v = self.v / numx.sqrt(self.d)
def symeig_semidefinite_reg(
A, B = None, eigenvectors=True, turbo="on", range=None,
type=1, overwrite=False, rank_threshold=1e-12, dfc_out=None):
"""
Regularization-based routine to solve generalized symmetric positive
semidefinite eigenvalue problems.
This can be used in case the normal symeig() call in _stop_training()
throws SymeigException ('Covariance matrices may be singular').
This solver applies a moderate regularization to B before applying
eigh/symeig. Afterwards it properly detects the rank deficit and
filters out malformed features.
For full range, this procedure is (approximately) as efficient as the
ordinary eigh implementation, because all additional steps are
computationally cheap.
For shorter range, the LDL method should be preferred.
The signature of this function equals that of mdp.utils.symeig, but
has two additional parameters:
rank_threshold: A threshold to determine if an eigenvalue counts as zero.
dfc_out: If dfc_out is not None dfc_out.rank_deficit will be set to an
integer indicating how many zero-eigenvalues were detected.
Note:
For efficiency reasons it actually modifies the matrix B
(even if overwrite=False), but the changes are negligible.
"""
if type != 1:
raise ValueError('Only type=1 is supported.')
# apply some regularization...
# The following is equivalent to B += 1e-12*np.eye(B.shape[0]),
# but works more in place, i.e. saves memory consumption of np.eye().
Bflat = B.reshape(B.shape[0]*B.shape[1])
idx = numx.arange(0, len(Bflat), B.shape[0]+1)
diag_tmp = Bflat[idx]
Bflat[idx] += rank_threshold
eg, ev = mdp.utils.symeig(A, B, True, turbo, None, type, overwrite)
Bflat[idx] = diag_tmp
m = numx.absolute(numx.sqrt(numx.absolute(
numx.sum(ev * mdp.utils.mult(B, ev), 0)))-1)
off = 0
# In theory all values in m should be close to one or close to zero.
# So we use the mean of these values as threshold to distinguish cases:
while m[off] > 0.5:
off += 1
m_off_sum = numx.sum(m[off:])
if m_off_sum < 0.5:
if off > 0:
if not dfc_out is None:
dfc_out.rank_deficit = off
eg = eg[off:]
ev = ev[:, off:]
else:
# Sometimes (unlikely though) the values in m are not sorted
# In this case we search all indices:
m_idx = (m < 0.5).nonzero()[0]
eg = eg[m_idx]
ev = ev[:, m_idx]
if range is None:
return eg, ev
else:
return eg[range[0]-1:range[1]], ev[:, range[0]-1:range[1]]
from past.utils import old_div
__docformat__ = "restructuredtext en"
import sys as _sys
import mdp
from mdp import Node, NodeException, numx, numx_rand
from mdp.nodes import WhiteningNode
from mdp.utils import (DelayCovarianceMatrix, MultipleCovarianceMatrices,
rotate, mult)
# Licensed under the BSD License, see Copyright file for details.
# TODO: support floats of size different than 64-bit; will need to change SQRT_EPS_D
# rename often used functions
sum, cos, sin, PI = numx.sum, numx.cos, numx.sin, numx.pi
SQRT_EPS_D = numx.sqrt(numx.finfo('d').eps)
def _triu(m, k=0):
"""Reduces a matrix to a triangular part.
:param m: A Matrix.
:param k: Index of diagonal.
:type k: int
:return: Elements on and above the k-th diagonal of m. k=0 is the
main diagonal, k > 0 is above and k < 0 is below the main diagonal.
"""
N = m.shape[0]
M = m.shape[1]
x = numx.greater_equal(numx.subtract.outer(numx.arange(N),
def symeig_semidefinite_pca(A,
B=None,
eigenvectors=True,
turbo="on",
range=None,
type=1,
overwrite=False,
rank_threshold=1e-12,
dfc_out=None):
"""
PCA-based routine to solve generalized symmetric positive semidefinite
eigenvalue problems.
This can be used if the normal ``symeig()`` call in ``_stop_training()``
throws ``SymeigException('Covariance matrices may be singular')``.
It applies PCA to B and filters out rank deficit before it applies
``symeig()`` to A.
It is roughly twice as expensive as the ordinary ``eigh`` implementation.
.. note::
The advantage compared to prepending a PCA node is that in execution
phase all data needs to be processed by one step less. That is because
this approach includes the PCA into e.g. the SFA execution matrix.
The signature of this function equals that of ``mdp.utils.symeig``,
but has two additional parameters:
:param rank_threshold:
A threshold to determine if an eigenvalue counts as zero.
:type rank_threshold: float
:param dfc_out:
If ``dfc_out`` is not ``None``, ``dfc_out.rank_deficit`` will be set
to an integer indicating how many zero-eigenvalues were detected.
"""
if type != 1:
raise ValueError('Only type=1 is supported.')
mult = mdp.utils.mult
# PCA-based method appears to be particularly unstable if blank lines
# and columns exist in B. So we circumvent this case:
nonzero_idx = _find_blank_data_idx(B, rank_threshold)
if not nonzero_idx is None:
orig_shape = B.shape
B = B[nonzero_idx, :][:, nonzero_idx]
A = A[nonzero_idx, :][:, nonzero_idx]
dcov_mtx = A
cov_mtx = B
eg, ev = mdp.utils.symeig(cov_mtx, None, True, turbo, None, type,
overwrite)
off = 0
while eg[off] < rank_threshold:
off += 1
if not dfc_out is None:
dfc_out.rank_deficit = off
eg = 1 / numx.sqrt(eg[off:])
ev2 = ev[:, off:]
ev2 *= eg
S = ev2
white = mult(S.T, mult(dcov_mtx, S))
eg, ev = mdp.utils.symeig(white, None, True, turbo, range, type, overwrite)
ev = mult(S, ev)
if not nonzero_idx is None:
# restore ev to original size
if not dfc_out is None:
dfc_out.rank_deficit += orig_shape[0] - len(nonzero_idx)
ev_tmp = ev
ev = numx.zeros((orig_shape[0], ev.shape[1]))
ev[nonzero_idx, :] = ev_tmp
return eg, ev
def _symeig_fake(A,
B=None,
eigenvectors=True,
turbo="on",
range=None,
type=1,
overwrite=False):
"""Solve standard and generalized eigenvalue problem for symmetric
(hermitian) definite positive matrices.
This function is a wrapper of LinearAlgebra.eigenvectors or
numarray.linear_algebra.eigenvectors with an interface compatible with symeig.
Syntax:
w,Z = symeig(A)
w = symeig(A,eigenvectors=0)
w,Z = symeig(A,range=(lo,hi))
w,Z = symeig(A,B,range=(lo,hi))
Inputs:
A -- An N x N matrix.
B -- An N x N matrix.
eigenvectors -- if set return eigenvalues and eigenvectors, otherwise
only eigenvalues
turbo -- not implemented
range -- the tuple (lo,hi) represent the indexes of the smallest and
largest (in ascending order) eigenvalues to be returned.
1 <= lo < hi <= N
if range = None, returns all eigenvalues and eigenvectors.
type -- not implemented, always solve A*x = (lambda)*B*x
overwrite -- not implemented
Outputs:
w -- (selected) eigenvalues in ascending order.
Z -- if range = None, Z contains the matrix of eigenvectors,
normalized as follows:
Z^H * A * Z = lambda and Z^H * B * Z = I
where ^H means conjugate transpose.
if range, an N x M matrix containing the orthonormal
eigenvectors of the matrix A corresponding to the selected
eigenvalues, with the i-th column of Z holding the eigenvector
associated with w[i]. The eigenvectors are normalized as above.
"""
dtype = numx.dtype(_greatest_common_dtype([A, B]))
try:
if B is None:
w, Z = numx_linalg.eigh(A)
else:
# make B the identity matrix
wB, ZB = numx_linalg.eigh(B)
_assert_eigenvalues_real_and_positive(wB, dtype)
ZB = old_div(ZB.real, numx.sqrt(wB.real))
# transform A in the new basis: A = ZB^T * A * ZB
A = mdp.utils.mult(mdp.utils.mult(ZB.T, A), ZB)
# diagonalize A
w, ZA = numx_linalg.eigh(A)
Z = mdp.utils.mult(ZB, ZA)
except numx_linalg.LinAlgError as exception:
raise SymeigException(str(exception))
_assert_eigenvalues_real_and_positive(w, dtype)
w = w.real
Z = Z.real
idx = w.argsort()
w = w.take(idx)
Z = Z.take(idx, axis=1)
# sanitize range:
n = A.shape[0]
if range is not None:
lo, hi = range
if lo < 1:
lo = 1
if lo > n:
lo = n
if hi > n:
hi = n
if lo > hi:
lo, hi = hi, lo
Z = Z[:, lo - 1:hi]
w = w[lo - 1:hi]
# the final call to refcast is necessary because of a bug in the casting
# behavior of Numeric and numarray: eigenvector does not wrap the LAPACK
# single precision routines
if eigenvectors:
return mdp.utils.refcast(w, dtype), mdp.utils.refcast(Z, dtype)
else:
return mdp.utils.refcast(w, dtype)
def _symeig_fake(A, B = None, eigenvectors = True, turbo = "on", range = None,
type = 1, overwrite = False):
"""Solve standard and generalized eigenvalue problem for symmetric
(hermitian) definite positive matrices.
This function is a wrapper of LinearAlgebra.eigenvectors or
numarray.linear_algebra.eigenvectors with an interface compatible with symeig.
Syntax:
w,Z = symeig(A)
w = symeig(A,eigenvectors=0)
w,Z = symeig(A,range=(lo,hi))
w,Z = symeig(A,B,range=(lo,hi))
Inputs:
A -- An N x N matrix.
B -- An N x N matrix.
eigenvectors -- if set return eigenvalues and eigenvectors, otherwise
only eigenvalues
turbo -- not implemented
range -- the tuple (lo,hi) represent the indexes of the smallest and
largest (in ascending order) eigenvalues to be returned.
1 <= lo < hi <= N
if range = None, returns all eigenvalues and eigenvectors.
type -- not implemented, always solve A*x = (lambda)*B*x
overwrite -- not implemented
Outputs:
w -- (selected) eigenvalues in ascending order.
Z -- if range = None, Z contains the matrix of eigenvectors,
normalized as follows:
Z^H * A * Z = lambda and Z^H * B * Z = I
where ^H means conjugate transpose.
if range, an N x M matrix containing the orthonormal
eigenvectors of the matrix A corresponding to the selected
eigenvalues, with the i-th column of Z holding the eigenvector
associated with w[i]. The eigenvectors are normalized as above.
"""
dtype = numx.dtype(_greatest_common_dtype([A, B]))
try:
if B is None:
w, Z = numx_linalg.eigh(A)
else:
# make B the identity matrix
wB, ZB = numx_linalg.eigh(B)
_assert_eigenvalues_real(wB, dtype)
if wB.real.min() < 0:
# If we proceeded with negative values here, this would let some
# NumPy or SciPy versions cause nan values in the results.
# Such nan values would go through silently (or only with a warning,
# i.e. RuntimeWarning: invalid value encountered in sqrt)
# and cause hard to find issues later in user code outside mdp.
err = "Got negative eigenvalues: %s" % str(wB)
raise SymeigException(err)
ZB = old_div(ZB.real, numx.sqrt(wB.real))
# transform A in the new basis: A = ZB^T * A * ZB
A = mdp.utils.mult(mdp.utils.mult(ZB.T, A), ZB)
# diagonalize A
w, ZA = numx_linalg.eigh(A)
Z = mdp.utils.mult(ZB, ZA)
except numx_linalg.LinAlgError as exception:
raise SymeigException(str(exception))
_assert_eigenvalues_real(w, dtype)
# Negative eigenvalues at this stage will be checked and handled by the caller.
w = w.real
Z = Z.real
idx = w.argsort()
w = w.take(idx)
Z = Z.take(idx, axis=1)
# sanitize range:
n = A.shape[0]
if range is not None:
lo, hi = range
if lo < 1:
lo = 1
if lo > n:
lo = n
if hi > n:
hi = n
if lo > hi:
lo, hi = hi, lo
Z = Z[:, lo-1:hi]
w = w[lo-1:hi]
# the final call to refcast is necessary because of a bug in the casting
# behavior of Numeric and numarray: eigenvector does not wrap the LAPACK
# single precision routines
if eigenvectors:
return mdp.utils.refcast(w, dtype), mdp.utils.refcast(Z, dtype)
else:
return mdp.utils.refcast(w, dtype)
def _stop_training(self):
Cumulator._stop_training(self)
k = self.k
M = self.data
N = M.shape[0]
if k > N:
err = ('k=%i must be less than'
' or equal to number of training points N=%i' % (k, N))
raise TrainingException(err)
if self.verbose:
print 'performing HLLE on %i points in %i dimensions...' % M.shape
# determines number of output dimensions: if desired_variance
# is specified, we need to learn it from the data. Otherwise,
# it's easy
learn_outdim = False
if self.output_dim is None:
if self.desired_variance is None:
self.output_dim = self.input_dim
else:
learn_outdim = True
# determine number of output dims, precalculate useful stuff
if learn_outdim:
Qs, sig2s, nbrss = self._adjust_output_dim()
d_out = self.output_dim
#dp = d_out + (d_out-1) + (d_out-2) + ...
dp = d_out * (d_out + 1) / 2
if min(k, N) <= d_out:
err = ('k=%i and n=%i (number of input data points) must be'
' larger than output_dim=%i' % (k, N, d_out))
raise TrainingException(err)
if k < 1 + d_out + dp:
wrn = ('The number of neighbours, k=%i, is smaller than'
' 1 + output_dim + output_dim*(output_dim+1)/2 = %i,'
' which might result in unstable results.' %
(k, 1 + d_out + dp))
_warnings.warn(wrn, MDPWarning)
#build the weight matrix
#XXX for faster implementation, W should be a sparse matrix
W = numx.zeros((N, dp * N), dtype=self.dtype)
if self.verbose:
print ' - constructing [%i x %i] weight matrix...' % W.shape
for row in range(N):
if learn_outdim:
nbrs = nbrss[row, :]
else:
# -----------------------------------------------
# find k nearest neighbors
# -----------------------------------------------
M_Mi = M - M[row]
nbrs = numx.argsort((M_Mi**2).sum(1))[1:k + 1]
#-----------------------------------------------
# center the neighborhood using the mean
#-----------------------------------------------
nbrhd = M[nbrs] # this makes a copy
nbrhd -= nbrhd.mean(0)
#-----------------------------------------------
# compute local coordinates
# using a singular value decomposition
#-----------------------------------------------
U, sig, VT = svd(nbrhd)
nbrhd = U.T[:d_out]
del VT
#-----------------------------------------------
# build Hessian estimator
#-----------------------------------------------
Yi = numx.zeros((dp, k), dtype=self.dtype)
ct = 0
for i in range(d_out):
Yi[ct:ct + d_out - i, :] = nbrhd[i] * nbrhd[i:, :]
ct += d_out - i
Yi = numx.concatenate(
[numx.ones((1, k), dtype=self.dtype), nbrhd, Yi], 0)
#-----------------------------------------------
# orthogonalize linear and quadratic forms
# with QR factorization
# and make the weights sum to 1
#-----------------------------------------------
if k >= 1 + d_out + dp:
Q, R = numx_linalg.qr(Yi.T)
w = Q[:, d_out + 1:d_out + 1 + dp]
else:
q, r = _mgs(Yi.T)
w = q[:, -dp:]
S = w.sum(0) #sum along columns
#if S[i] is too small, set it equal to 1.0
# this prevents weights from blowing up
S[numx.where(numx.absolute(S) < 1E-4)] = 1.0
#print w.shape, S.shape, (w/S).shape
#print W[nbrs, row*dp:(row+1)*dp].shape
W[nbrs, row * dp:(row + 1) * dp] = w / S
#-----------------------------------------------
# To find the null space, we want the
# first d+1 eigenvectors of W.T*W
# Compute this using an svd of W
#-----------------------------------------------
if self.verbose:
msg = (' - finding [%i x %i] '
'null space of weight matrix...' % (d_out, N))
print msg
#XXX future work:
#XXX use of upcoming ARPACK interface for bottom few eigenvectors
#XXX of a sparse matrix will significantly increase the speed
#XXX of the next step
if self.svd:
sig, U = nongeneral_svd(W.T, range=(2, d_out + 1))
Y = U * numx.sqrt(N)
else:
WW = mult(W, W.T)
# regularizes the eigenvalues, does not change the eigenvectors:
W_diag_idx = numx.arange(N)
WW[W_diag_idx, W_diag_idx] += 0.01
sig, U = symeig(WW, range=(2, self.output_dim + 1), overwrite=True)
Y = U * numx.sqrt(N)
del WW
del W
#-----------------------------------------------
# Normalize Y
#
# Alternative way to do it:
# we need R = (Y.T*Y)^(-1/2)
# do this with an SVD of Y del VT
# Y = U*sig*V.T
# Y.T*Y = (V*sig.T*U.T) * (U*sig*V.T)
# = V * (sig*sig.T) * V.T
# = V * sig^2 V.T
# so
# R = V * sig^-1 * V.T
# The code is:
# U, sig, VT = svd(Y)
# del U
# S = numx.diag(sig**-1)
# self.training_projection = mult(Y, mult(VT.T, mult(S, VT)))
#-----------------------------------------------
if self.verbose:
print ' - normalizing null space...'
C = sqrtm(mult(Y.T, Y))
self.training_projection = mult(Y, C)
def symeig_semidefinite_ldl(A,
B=None,
eigenvectors=True,
turbo="on",
rng=None,
type=1,
overwrite=False,
rank_threshold=1e-12,
dfc_out=None):
"""LDL-based routine to solve generalized symmetric positive semidefinite
eigenvalue problems.
This can be used if the normal ``symeig()`` call in ``_stop_training()``
throws ``SymeigException('Covariance matrices may be singular')``.
This solver uses SciPy's raw LAPACK interface to access LDL decomposition.
http://www.netlib.org/lapack/lug/node54.html describes how to solve a
generalized eigenvalue problem with positive definite B using Cholesky/LL
decomposition. We extend this method to solve for positive semidefinite B
using LDL decomposition, which is a variant of Cholesky/LL decomposition
for indefinite Matrices.
Accessing raw LAPACK's LDL decomposition (sytrf) is challenging. This code
is partly based on code for SciPy 1.1:
http://github.com/scipy/scipy/pull/7941/files#diff-9bf9b4b2f0f40415bc0e72143584c889
We optimized and shortened that code for the real-valued positive
semidefinite case.
This procedure is almost as efficient as the ordinary eigh implementation.
This is because implementations for symmetric generalized eigenvalue
problems usually perform the Cholesky approach mentioned above. The more
general LDL decomposition is only slightly more expensive than Cholesky,
due to pivotization.
.. note:: This method requires SciPy >= 1.0.
The signature of this function equals that of ``mdp.utils.symeig``,
but has two additional parameters:
:param rank_threshold:
A threshold to determine if an eigenvalue counts as zero.
:type rank_threshold: float
:param dfc_out:
If ``dfc_out`` is not ``None``, ``dfc_out.rank_deficit`` will be set
to an integer indicating how many zero-eigenvalues were detected.
"""
if type != 1:
raise ValueError('Only type=1 is supported.')
# LDL-based method appears to be particularly unstable if blank lines
# and columns exist in B. So we circumvent this case:
nonzero_idx = _find_blank_data_idx(B, rank_threshold)
if not nonzero_idx is None:
orig_shape = B.shape
B = B[nonzero_idx, :][:, nonzero_idx]
A = A[nonzero_idx, :][:, nonzero_idx]
# This method has special requirements, which is why we import here
# rather than module wide.
from scipy.linalg.lapack import get_lapack_funcs, _compute_lwork
from scipy.linalg.blas import get_blas_funcs
try:
inv_tri, solver, solver_lwork = get_lapack_funcs(
('trtri', 'sytrf', 'sytrf_lwork'), (B, ))
mult_tri, = get_blas_funcs(('trmm', ), (B, ))
except ValueError:
err_msg = ("ldl method for solving symeig with rank deficit B "
"requires at least SciPy 1.0.")
raise SymeigException(err_msg)
n = B.shape[0]
arng = numx.arange(n)
lwork = _compute_lwork(solver_lwork, n, lower=1)
lu, piv, _ = solver(B, lwork=lwork, lower=1, overwrite_a=overwrite)
# using piv properly requires some postprocessing:
swap_ = numx.arange(n)
pivs = numx.zeros(swap_.shape, dtype=int)
skip_2x2 = False
for ind in range(n):
# If previous spin belonged already to a 2x2 block
if skip_2x2:
skip_2x2 = False
continue
cur_val = piv[ind]
# do we have a 1x1 block or not?
if cur_val > 0:
if cur_val != ind + 1:
# Index value != array value --> permutation required
swap_[ind] = swap_[cur_val - 1]
pivs[ind] = 1
# Not.
elif cur_val < 0 and cur_val == piv[ind + 1]:
# first neg entry of 2x2 block identifier
if -cur_val != ind + 2:
# Index value != array value --> permutation required
swap_[ind + 1] = swap_[-cur_val - 1]
pivs[ind] = 2
skip_2x2 = True
full_perm = numx.arange(n)
for ind in range(n - 1, -1, -1):
s_ind = swap_[ind]
if s_ind != ind:
col_s = ind if pivs[ind] else ind - 1 # 2x2 block
lu[[s_ind, ind], col_s:] = lu[[ind, s_ind], col_s:]
full_perm[[s_ind, ind]] = full_perm[[ind, s_ind]]
# usually only a few indices actually permute, so we reduce perm:
perm = (full_perm - arng).nonzero()[0]
perm_idx = full_perm[perm]
# end of ldl postprocessing
# perm_idx and perm now describe a permutation as dest and source indexes
lu[perm_idx, :] = lu[perm, :]
dgd = abs(numx.diag(lu))
dnz = (dgd > rank_threshold).nonzero()[0]
dgd_sqrt_I = numx.sqrt(1.0 / dgd[dnz])
rank_deficit = len(dgd) - len(dnz) # later used
# c, lower, unitdiag, overwrite_c
LI, _ = inv_tri(lu, 1, 1, 1) # invert triangular
# we mainly apply tril here, because we need to make a
# copy of LI anyway, because original result from
# dtrtri seems to be read-only regarding some operations
LI = numx.tril(LI, -1)
LI[arng, arng] = 1
LI[dnz, :] *= dgd_sqrt_I.reshape((dgd_sqrt_I.shape[0], 1))
A2 = A if overwrite else A.copy()
A2[perm_idx, :] = A2[perm, :]
A2[:, perm_idx] = A2[:, perm]
# alpha, a, b, side 0=left 1=right, lower, trans_a, diag 1=unitdiag,
# overwrite_b
A2 = mult_tri(1.0, LI, A2, 1, 1, 1, 0, 1) # A2 = mult(A2, LI.T)
A2 = mult_tri(1.0, LI, A2, 0, 1, 0, 0, 1) # A2 = mult(LI, A2)
A2 = A2[dnz, :]
A2 = A2[:, dnz]
# overwrite=True is okay here, because at this point A2 is a copy anyway
eg, ev = mdp.utils.symeig(A2, None, True, turbo, rng, overwrite=True)
ev = mdp.utils.mult(LI[dnz].T, ev) if rank_deficit \
else mult_tri(1.0, LI, ev, 0, 1, 1, 0, 1)
ev[perm] = ev[perm_idx]
if not nonzero_idx is None:
# restore ev to original size
rank_deficit += orig_shape[0] - len(nonzero_idx)
ev_tmp = ev
ev = numx.zeros((orig_shape[0], ev.shape[1]))
ev[nonzero_idx, :] = ev_tmp
if not dfc_out is None:
dfc_out.rank_deficit = rank_deficit
return eg, ev
def _stop_training(self):
Cumulator._stop_training(self)
k = self.k
M = self.data
N = M.shape[0]
if k > N:
err = ('k=%i must be less than'
' or equal to number of training points N=%i' % (k, N))
raise TrainingException(err)
if self.verbose:
print 'performing HLLE on %i points in %i dimensions...' % M.shape
# determines number of output dimensions: if desired_variance
# is specified, we need to learn it from the data. Otherwise,
# it's easy
learn_outdim = False
if self.output_dim is None:
if self.desired_variance is None:
self.output_dim = self.input_dim
else:
learn_outdim = True
# determine number of output dims, precalculate useful stuff
if learn_outdim:
Qs, sig2s, nbrss = self._adjust_output_dim()
d_out = self.output_dim
#dp = d_out + (d_out-1) + (d_out-2) + ...
dp = d_out*(d_out+1)/2
if min(k, N) <= d_out:
err = ('k=%i and n=%i (number of input data points) must be'
' larger than output_dim=%i' % (k, N, d_out))
raise TrainingException(err)
if k < 1+d_out+dp:
wrn = ('The number of neighbours, k=%i, is smaller than'
' 1 + output_dim + output_dim*(output_dim+1)/2 = %i,'
' which might result in unstable results.'
% (k, 1+d_out+dp))
_warnings.warn(wrn, MDPWarning)
#build the weight matrix
#XXX for faster implementation, W should be a sparse matrix
W = numx.zeros((N, dp*N), dtype=self.dtype)
if self.verbose:
print ' - constructing [%i x %i] weight matrix...' % W.shape
for row in range(N):
if learn_outdim:
nbrs = nbrss[row, :]
else:
# -----------------------------------------------
# find k nearest neighbors
# -----------------------------------------------
M_Mi = M-M[row]
nbrs = numx.argsort((M_Mi**2).sum(1))[1:k+1]
#-----------------------------------------------
# center the neighborhood using the mean
#-----------------------------------------------
nbrhd = M[nbrs] # this makes a copy
nbrhd -= nbrhd.mean(0)
#-----------------------------------------------
# compute local coordinates
# using a singular value decomposition
#-----------------------------------------------
U, sig, VT = svd(nbrhd)
nbrhd = U.T[:d_out]
del VT
#-----------------------------------------------
# build Hessian estimator
#-----------------------------------------------
Yi = numx.zeros((dp, k), dtype=self.dtype)
ct = 0
for i in range(d_out):
Yi[ct:ct+d_out-i, :] = nbrhd[i] * nbrhd[i:, :]
ct += d_out-i
Yi = numx.concatenate([numx.ones((1, k), dtype=self.dtype),
nbrhd, Yi], 0)
#-----------------------------------------------
# orthogonalize linear and quadratic forms
# with QR factorization
# and make the weights sum to 1
#-----------------------------------------------
if k >= 1+d_out+dp:
Q, R = numx_linalg.qr(Yi.T)
w = Q[:, d_out+1:d_out+1+dp]
else:
q, r = _mgs(Yi.T)
w = q[:, -dp:]
S = w.sum(0) #sum along columns
#if S[i] is too small, set it equal to 1.0
# this prevents weights from blowing up
S[numx.where(numx.absolute(S)<1E-4)] = 1.0
#print w.shape, S.shape, (w/S).shape
#print W[nbrs, row*dp:(row+1)*dp].shape
W[nbrs, row*dp:(row+1)*dp] = w / S
#-----------------------------------------------
# To find the null space, we want the
# first d+1 eigenvectors of W.T*W
# Compute this using an svd of W
#-----------------------------------------------
if self.verbose:
msg = (' - finding [%i x %i] '
'null space of weight matrix...' % (d_out, N))
print msg
#XXX future work:
#XXX use of upcoming ARPACK interface for bottom few eigenvectors
#XXX of a sparse matrix will significantly increase the speed
#XXX of the next step
if self.svd:
sig, U = nongeneral_svd(W.T, range=(2, d_out+1))
Y = U*numx.sqrt(N)
else:
WW = mult(W, W.T)
# regularizes the eigenvalues, does not change the eigenvectors:
W_diag_idx = numx.arange(N)
WW[W_diag_idx, W_diag_idx] += 0.01
sig, U = symeig(WW, range=(2, self.output_dim+1), overwrite=True)
Y = U*numx.sqrt(N)
del WW
del W
#-----------------------------------------------
# Normalize Y
#
# Alternative way to do it:
# we need R = (Y.T*Y)^(-1/2)
# do this with an SVD of Y del VT
# Y = U*sig*V.T
# Y.T*Y = (V*sig.T*U.T) * (U*sig*V.T)
# = V * (sig*sig.T) * V.T
# = V * sig^2 V.T
# so
# R = V * sig^-1 * V.T
# The code is:
# U, sig, VT = svd(Y)
# del U
# S = numx.diag(sig**-1)
# self.training_projection = mult(Y, mult(VT.T, mult(S, VT)))
#-----------------------------------------------
if self.verbose:
print ' - normalizing null space...'
C = sqrtm(mult(Y.T, Y))
self.training_projection = mult(Y, C)
def symeig_semidefinite_reg(A,
B=None,
eigenvectors=True,
turbo="on",
range=None,
type=1,
overwrite=False,
rank_threshold=1e-12,
dfc_out=None):
"""Regularization-based routine to solve generalized symmetric positive
semidefinite eigenvalue problems.
This can be used if the normal ``symeig()`` call in ``_stop_training()``
throws ``SymeigException('Covariance matrices may be singular')``.
This solver applies a moderate regularization to B before applying
``eigh``/``symeig``. Afterwards it properly detects the rank deficit and
filters out malformed features.
For full range, this procedure is (approximately) as efficient as the
ordinary ``eigh`` implementation, because all additional steps are
computationally cheap.
For shorter range, the LDL method should be preferred.
.. note::
For efficiency reasons it actually modifies the matrix B
(even if ``overwrite=False``), but the changes are negligible.
The signature of this function equals that of ``mdp.utils.symeig``,
but has two additional parameters:
:param rank_threshold:
A threshold to determine if an eigenvalue counts as zero.
:type rank_threshold: float
:param dfc_out:
If ``dfc_out`` is not ``None``, ``dfc_out.rank_deficit`` will be set
to an integer indicating how many zero-eigenvalues were detected.
"""
if type != 1:
raise ValueError('Only type=1 is supported.')
# apply some regularization...
# The following is equivalent to B += 1e-12*np.eye(B.shape[0]),
# but works more in place, i.e. saves memory consumption of np.eye().
Bflat = B.reshape(B.shape[0] * B.shape[1])
idx = numx.arange(0, len(Bflat), B.shape[0] + 1)
diag_tmp = Bflat[idx]
Bflat[idx] += rank_threshold
eg, ev = mdp.utils.symeig(A, B, True, turbo, None, type, overwrite)
Bflat[idx] = diag_tmp
m = numx.absolute(
numx.sqrt(numx.absolute(numx.sum(ev * mdp.utils.mult(B, ev), 0))) - 1)
off = 0
# In theory all values in m should be close to one or close to zero.
# So we use the mean of these values as threshold to distinguish cases:
while m[off] > 0.5:
off += 1
m_off_sum = numx.sum(m[off:])
if m_off_sum < 0.5:
if off > 0:
if not dfc_out is None:
dfc_out.rank_deficit = off
eg = eg[off:]
ev = ev[:, off:]
else:
# Sometimes (unlikely though) the values in m are not sorted
# In this case we search all indices:
m_idx = (m < 0.5).nonzero()[0]
eg = eg[m_idx]
ev = ev[:, m_idx]
if range is None:
return eg, ev
else:
return eg[range[0] - 1:range[1]], ev[:, range[0] - 1:range[1]]
from past.utils import old_div
__docformat__ = "restructuredtext en"
import sys as _sys
import mdp
from mdp import Node, NodeException, numx, numx_rand
from mdp.nodes import WhiteningNode
from mdp.utils import (DelayCovarianceMatrix, MultipleCovarianceMatrices,
rotate, mult)
# TODO: support floats of size different than 64-bit; will need to change SQRT_EPS_D
# rename often used functions
sum, cos, sin, PI = numx.sum, numx.cos, numx.sin, numx.pi
SQRT_EPS_D = numx.sqrt(numx.finfo('d').eps)
def _triu(m, k=0):
""" returns the elements on and above the k-th diagonal of m. k=0 is the
main diagonal, k > 0 is above and k < 0 is below the main diagonal."""
N = m.shape[0]
M = m.shape[1]
x = numx.greater_equal(numx.subtract.outer(numx.arange(N),
numx.arange(M)),1-k)
out = (1-x)*m
return out
#############
class ISFANode(Node):
"""
Perform Independent Slow Feature Analysis on the input data.
def symeig_semidefinite_ldl(
A, B = None, eigenvectors=True, turbo="on", rng=None,
type=1, overwrite=False, rank_threshold=1e-12, dfc_out=None):
"""
LDL-based routine to solve generalized symmetric positive semidefinite
eigenvalue problems.
This can be used in case the normal symeig() call in _stop_training()
throws SymeigException ('Covariance matrices may be singular').
This solver uses SciPy's raw LAPACK interface to access LDL decomposition.
www.netlib.org/lapack/lug/node54.html describes how to solve a
generalized eigenvalue problem with positive definite B using Cholesky/LL
decomposition. We extend this method to solve for positive semidefinite B
using LDL decomposition, which is a variant of Cholesky/LL decomposition
for indefinite Matrices.
Accessing raw LAPACK's LDL decomposition (sytrf) is challenging. This code
is partly based on code for SciPy 1.1:
github.com/scipy/scipy/pull/7941/files#diff-9bf9b4b2f0f40415bc0e72143584c889
We optimized and shortened that code for the real-valued positive
semidefinite case.
This procedure is almost as efficient as the ordinary eigh implementation.
This is because implementations for symmetric generalized eigenvalue
problems usually perform the Cholesky approach mentioned above. The more
general LDL decomposition is only slightly more expensive than Cholesky,
due to pivotization.
The signature of this function equals that of mdp.utils.symeig, but
has two additional parameters:
rank_threshold: A threshold to determine if an eigenvalue counts as zero.
dfc_out: If dfc_out is not None dfc_out.rank_deficit will be set to an
integer indicating how many zero-eigenvalues were detected.
Note:
This method requires SciPy >= 1.0.
"""
if type != 1:
raise ValueError('Only type=1 is supported.')
# LDL-based method appears to be particularly unstable if blank lines
# and columns exist in B. So we circumvent this case:
nonzero_idx = _find_blank_data_idx(B, rank_threshold)
if not nonzero_idx is None:
orig_shape = B.shape
B = B[nonzero_idx, :][:, nonzero_idx]
A = A[nonzero_idx, :][:, nonzero_idx]
# This method has special requirements, which is why we import here
# rather than module wide.
from scipy.linalg.lapack import get_lapack_funcs, _compute_lwork
from scipy.linalg.blas import get_blas_funcs
try:
inv_tri, solver, solver_lwork = get_lapack_funcs(
('trtri', 'sytrf', 'sytrf_lwork'), (B,))
mult_tri, = get_blas_funcs(('trmm',), (B,))
except ValueError:
err_msg = ("ldl method for solving symeig with rank deficit B "
"requires at least SciPy 1.0.")
raise SymeigException(err_msg)
n = B.shape[0]
arng = numx.arange(n)
lwork = _compute_lwork(solver_lwork, n, lower=1)
lu, piv, _ = solver(B, lwork=lwork, lower=1, overwrite_a=overwrite)
# using piv properly requires some postprocessing:
swap_ = numx.arange(n)
pivs = numx.zeros(swap_.shape, dtype=int)
skip_2x2 = False
for ind in range(n):
# If previous spin belonged already to a 2x2 block
if skip_2x2:
skip_2x2 = False
continue
cur_val = piv[ind]
# do we have a 1x1 block or not?
if cur_val > 0:
if cur_val != ind+1:
# Index value != array value --> permutation required
swap_[ind] = swap_[cur_val-1]
pivs[ind] = 1
# Not.
elif cur_val < 0 and cur_val == piv[ind+1]:
# first neg entry of 2x2 block identifier
if -cur_val != ind+2:
# Index value != array value --> permutation required
swap_[ind+1] = swap_[-cur_val-1]
pivs[ind] = 2
skip_2x2 = True
full_perm = numx.arange(n)
for ind in range(n-1, -1, -1):
s_ind = swap_[ind]
if s_ind != ind:
col_s = ind if pivs[ind] else ind-1 # 2x2 block
lu[[s_ind, ind], col_s:] = lu[[ind, s_ind], col_s:]
full_perm[[s_ind, ind]] = full_perm[[ind, s_ind]]
# usually only a few indices actually permute, so we reduce perm:
perm = (full_perm-arng).nonzero()[0]
perm_idx = full_perm[perm]
# end of ldl postprocessing
# perm_idx and perm now describe a permutation as dest and source indexes
lu[perm_idx, :] = lu[perm, :]
dgd = abs(numx.diag(lu))
dnz = (dgd > rank_threshold).nonzero()[0]
dgd_sqrt_I = numx.sqrt(1.0/dgd[dnz])
rank_deficit = len(dgd) - len(dnz) # later used
# c, lower, unitdiag, overwrite_c
LI, _ = inv_tri(lu, 1, 1, 1) # invert triangular
# we mainly apply tril here, because we need to make a
# copy of LI anyway, because original result from
# dtrtri seems to be read-only regarding some operations
LI = numx.tril(LI, -1)
LI[arng, arng] = 1
LI[dnz, :] *= dgd_sqrt_I.reshape((dgd_sqrt_I.shape[0], 1))
A2 = A if overwrite else A.copy()
A2[perm_idx, :] = A2[perm, :]
A2[:, perm_idx] = A2[:, perm]
# alpha, a, b, side 0=left 1=right, lower, trans_a, diag 1=unitdiag,
# overwrite_b
A2 = mult_tri(1.0, LI, A2, 1, 1, 1, 0, 1) # A2 = mult(A2, LI.T)
A2 = mult_tri(1.0, LI, A2, 0, 1, 0, 0, 1) # A2 = mult(LI, A2)
A2 = A2[dnz, :]
A2 = A2[:, dnz]
# overwrite=True is okay here, because at this point A2 is a copy anyway
eg, ev = mdp.utils.symeig(A2, None, True, turbo, rng, overwrite=True)
ev = mdp.utils.mult(LI[dnz].T, ev) if rank_deficit \
else mult_tri(1.0, LI, ev, 0, 1, 1, 0, 1)
ev[perm] = ev[perm_idx]
if not nonzero_idx is None:
# restore ev to original size
rank_deficit += orig_shape[0]-len(nonzero_idx)
ev_tmp = ev
ev = numx.zeros((orig_shape[0], ev.shape[1]))
ev[nonzero_idx, :] = ev_tmp
if not dfc_out is None:
dfc_out.rank_deficit = rank_deficit
return eg, ev
def symeig_semidefinite_pca(
A, B = None, eigenvectors=True, turbo="on", range=None,
type=1, overwrite=False, rank_threshold=1e-12, dfc_out=None):
"""
PCA-based routine to solve generalized symmetric positive semidefinite
eigenvalue problems.
This can be used in case the normal symeig() call in _stop_training()
throws SymeigException ('Covariance matrices may be singular').
It applies PCA to B and filters out rank deficit before it applies
symeig() to A.
It is roughly twice as expensive as the ordinary eigh implementation.
The signature of this function equals that of mdp.utils.symeig, but
has two additional parameters:
rank_threshold: A threshold to determine if an eigenvalue counts as zero.
dfc_out: If dfc_out is not None dfc_out.rank_deficit will be set to an
integer indicating how many zero-eigenvalues were detected.
Note:
The advantage compared to prepending a PCA node is that in execution
phase all data needs to be processed by one step less. That is because
this approach includes the PCA into e.g. the SFA execution matrix.
"""
if type != 1:
raise ValueError('Only type=1 is supported.')
mult = mdp.utils.mult
# PCA-based method appears to be particularly unstable if blank lines
# and columns exist in B. So we circumvent this case:
nonzero_idx = _find_blank_data_idx(B, rank_threshold)
if not nonzero_idx is None:
orig_shape = B.shape
B = B[nonzero_idx, :][:, nonzero_idx]
A = A[nonzero_idx, :][:, nonzero_idx]
dcov_mtx = A
cov_mtx = B
eg, ev = mdp.utils.symeig(cov_mtx, None, True, turbo, None, type,
overwrite)
off = 0
while eg[off] < rank_threshold:
off += 1
if not dfc_out is None:
dfc_out.rank_deficit = off
eg = 1/numx.sqrt(eg[off:])
ev2 = ev[:, off:]
ev2 *= eg
S = ev2
white = mult(S.T, mult(dcov_mtx, S))
eg, ev = mdp.utils.symeig(white, None, True, turbo, range, type,
overwrite)
ev = mult(S, ev)
if not nonzero_idx is None:
# restore ev to original size
if not dfc_out is None:
dfc_out.rank_deficit += orig_shape[0]-len(nonzero_idx)
ev_tmp = ev
ev = numx.zeros((orig_shape[0], ev.shape[1]))
ev[nonzero_idx, :] = ev_tmp
return eg, ev
def _symeig_fake(A,
B=None,
eigenvectors=True,
turbo="on",
range=None,
type=1,
overwrite=False):
"""Solve standard and generalized eigenvalue problem for symmetric
(hermitian) definite positive matrices.
This function is a wrapper of LinearAlgebra.eigenvectors or
numarray.linear_algebra.eigenvectors with an interface compatible with symeig.
Syntax:
w,Z = symeig(A)
w = symeig(A,eigenvectors=0)
w,Z = symeig(A,range=(lo,hi))
w,Z = symeig(A,B,range=(lo,hi))
Inputs:
A -- An N x N matrix.
B -- An N x N matrix.
eigenvectors -- if set return eigenvalues and eigenvectors, otherwise
only eigenvalues
turbo -- not implemented
range -- the tuple (lo,hi) represent the indexes of the smallest and
largest (in ascending order) eigenvalues to be returned.
1 <= lo < hi <= N
if range = None, returns all eigenvalues and eigenvectors.
type -- not implemented, always solve A*x = (lambda)*B*x
overwrite -- not implemented
Outputs:
w -- (selected) eigenvalues in ascending order.
Z -- if range = None, Z contains the matrix of eigenvectors,
normalized as follows:
Z^H * A * Z = lambda and Z^H * B * Z = I
where ^H means conjugate transpose.
if range, an N x M matrix containing the orthonormal
eigenvectors of the matrix A corresponding to the selected
eigenvalues, with the i-th column of Z holding the eigenvector
associated with w[i]. The eigenvectors are normalized as above.
"""
dtype = numx.dtype(_greatest_common_dtype([A, B]))
try:
if B is None:
w, Z = numx_linalg.eigh(A)
else:
# make B the identity matrix
wB, ZB = numx_linalg.eigh(B)
_assert_eigenvalues_real_and_positive(wB, dtype)
ZB = ZB.real / numx.sqrt(wB.real)
# transform A in the new basis: A = ZB^T * A * ZB
A = mdp.utils.mult(mdp.utils.mult(ZB.T, A), ZB)
# diagonalize A
w, ZA = numx_linalg.eigh(A)
Z = mdp.utils.mult(ZB, ZA)
except numx_linalg.LinAlgError, exception:
raise SymeigException(str(exception))
__docformat__ = "restructuredtext en"
import sys as _sys
import mdp
from mdp import Node, NodeException, numx, numx_rand
from mdp.nodes import WhiteningNode
from mdp.utils import DelayCovarianceMatrix, MultipleCovarianceMatrices, rotate, mult
# TODO: support floats of size different than 64-bit; will need to change SQRT_EPS_D
# rename often used functions
sum, cos, sin, PI = numx.sum, numx.cos, numx.sin, numx.pi
SQRT_EPS_D = numx.sqrt(numx.finfo("d").eps)
def _triu(m, k=0):
""" returns the elements on and above the k-th diagonal of m. k=0 is the
main diagonal, k > 0 is above and k < 0 is below the main diagonal."""
N = m.shape[0]
M = m.shape[1]
x = numx.greater_equal(numx.subtract.outer(numx.arange(N), numx.arange(M)), 1 - k)
out = (1 - x) * m
return out
#############
class ISFANode(Node):
"""
Perform Independent Slow Feature Analysis on the input data.
