def jitchol(A, maxtries=5):
A = np.ascontiguousarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise linalg.LinAlgError("not pd: non-positive diagonal elements")
jitter = diagA.mean() * 1e-6
num_tries = 1
while num_tries <= maxtries and np.isfinite(jitter):
try:
L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter,
lower=True)
return L
except:
jitter *= 10
finally:
num_tries += 1
raise linalg.LinAlgError("not positive definite, even with jitter.")
import traceback
try:
raise
except:
logging.warning('\n'.join([
'Added jitter of {:.10e}'.format(jitter),
' in ' + traceback.format_list(
traceback.extract_stack(limit=3)[-2:-1])[0][2:]
]))
return L
def calc_invsqrt(self, hessian):
"""Calculate the inverse square root, given the Hessian matrix
:param hessian: Hessian matrix of the parameter space
"""
try:
# Try Cholesky
self.ch = sl.cholesky(hessian, lower=True)
# Fast solve
self.chi = sl.solve_triangular(self.ch, np.eye(len(self.ch)),
trans=0, lower=True)
self.lj = np.sum(np.log(np.diag(self.chi)))
except sl.LinAlgError:
# Cholesky fails. Try eigh
try:
eigval, eigvec = sl.eigh(hessian)
if not np.all(eigval > 0):
# Try SVD here? Or just regularize?
raise sl.LinAlgError("Eigh thinks hessian is not positive definite")
self.ch = eigvec * np.sqrt(eigval)
self.chi = (eigvec / np.sqrt(eigval)).T
self.lj = -0.5*np.sum(np.log(eigval))
except sl.LinAlgError:
U, s, Vt = sl.svd(hessian)
if not np.all(s > 0):
raise sl.LinAlgError("SVD thinks hessian is not positive definite")
self.ch = U * np.sqrt(s)
self.chi = (U / np.sqrt(s)).T
self.lj = -0.5*np.sum(np.log(s))
def cholesky(cov, max_tries=5):
"""
Computest the Cholesky decomposition L of the matrix cov: L*L^T = cov
:param cov: np.array(nxn)
:param max_tries: int
:return: L
"""
cov = np.ascontiguousarray(cov)
L, info = lapack.dpotrf(cov, lower=1)
if info == 0:
return L
diag_cov = np.diag(cov)
if np.any(diag_cov <= 0.):
raise linalg.LinAlgError("not positive definite matrix")
jitter = diag_cov.mean() * 1e-6
n_tries = 1
while n_tries <= max_tries and np.isfinite(jitter):
try:
L = linalg.cholesky(cov + np.eye(cov.shape[0]) * jitter,
lower=True)
return L
except:
jitter *= 10
finally:
n_tries += 1
raise linalg.LinAlgError("not positive definite, even with jitter.")
def check_cholesky_inputs(A):
"""
Check inputs to cholesky routines
This function is used by both specialized Cholesky routines to check inputs. It verifies
that the input is a 2D square matrix and that all diagonal elements are positive (a
necessary but not sufficient condition for positive definiteness). If these checks pass,
it returns the input as a numpy array.
:param A: The matrix to be inverted as an array of shape ``(n,n)``. Must be a symmetric positive
definite matrix.
:type A: ndarray or similar
:returns: The matrix to be inverted as an array of shape ``(n,n)``. Must be a symmetric positive
definite matrix.
:rtype: ndarray
"""
A = np.array(A)
assert A.ndim == 2, "A must have shape (n,n)"
assert A.shape[0] == A.shape[1], "A must have shape (n,n)"
np.testing.assert_allclose(A.T, A)
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise linalg.LinAlgError("not pd: non-positive diagonal elements")
return A
def choleskyjitter(A, overwrite_a=False, check_finite=True):
"""Add jitter stochastically until a positive definite matrix occurs"""
# Avoid preparing for jittering if we can already find the cholesky
# with no problem
try:
return la.cholesky(A,
lower=True,
overwrite_a=overwrite_a,
check_finite=check_finite)
except Exception:
pass
# Prepare for jittering (all the magic numbers here are arbitary...)
n = A.shape[0]
maxscale = 1e10
minscale = 1e-4
scale = minscale
# Keep jittering stochastically, increasing the jitter magnitude along
# the way, until it's all good
while scale < maxscale:
try:
jitA = scale * np.diag(np.random.rand(n))
L = la.cholesky(A + jitA,
lower=True,
overwrite_a=overwrite_a,
check_finite=check_finite)
return L
except la.LinAlgError:
scale *= 1.01
log.warning('Jitter added stochastically. Scale: %f!' % scale)
raise la.LinAlgError("Max value of jitter reached")
def _do_pre_eig(self, m, b, k):
"""Do a "pre" eigensolution to put system in modal space"""
if k.ndim == 1:
k = np.diag(k)
else:
ktype, types = ytools.mattype(k)
if not ((ktype & types["symmetric"]) or
(ktype & types["hermitian"])):
raise la.LinAlgError(
"stiffness matrix must be symmetric or hermitian for the "
"`pre_eig` option.")
if m is None:
w, u = la.eigh(k)
else:
if m.ndim == 1:
m = np.diag(m)
w, u = la.eigh(k, m)
self.pre_eig = True
self.phi = u
m = None
k = w
if b.ndim == 1:
b = (u.T * b) @ u
else:
b = u.T @ b @ u
return m, b, k
def __init__(self, dataset, bw_method=None, weights=None):
self.dataset = atleast_2d(asarray(dataset))
if not self.dataset.size > 1:
raise ValueError("`dataset` input should have multiple elements.")
self.d, self.n = self.dataset.shape
if weights is not None:
self._weights = atleast_1d(weights).astype(float)
self._weights /= sum(self._weights)
if self.weights.ndim != 1:
raise ValueError("`weights` input should be one-dimensional.")
if len(self._weights) != self.n:
raise ValueError("`weights` input should be of length n")
self._neff = 1 / sum(self._weights**2)
try:
self.set_bandwidth(bw_method=bw_method)
except linalg.LinAlgError as e:
msg = ("The data appears to lie in a lower-dimensional subspace "
"of the space in which it is expressed. This has resulted "
"in a singular data covariance matrix, which cannot be "
"treated using the algorithms implemented in "
"`gaussian_kde`. Consider performing principle component "
"analysis / dimensionality reduction and using "
"`gaussian_kde` with the transformed data.")
raise linalg.LinAlgError(msg) from e
def jitchol(X, overwrite_a=False, check_finite=True):
"""Add jitter until a positive definite matrix occurs"""
n = X.shape[0]
I = np.eye(n)
jitter = 1e-8
max_jitter = 1e10
L = None
X_dash = X
while jitter < max_jitter:
try:
L = la.cholesky(X_dash,
lower=True,
overwrite_a=overwrite_a,
check_finite=check_finite)
break
except la.LinAlgError:
X_dash = X + jitter * I
jitter *= 2.0
log.warning('Jitter added. Amount: %f!' % jitter)
if jitter > 1e-2:
log.warning('Rather large jitchol of %f!' % jitter)
if L is not None:
return L
else:
raise la.LinAlgError("Max value of jitter reached")
def jitChol(A, maxTries=10, warning=True):
"""Do a Cholesky decomposition with jitter.
Description:
U, jitter = jitChol(A, maxTries, warning) attempts a Cholesky
decomposition on the given matrix, if matrix isn't positive
definite the function adds 'jitter' and tries again. Thereafter
the amount of jitter is multiplied by 10 each time it is added
again. This is continued for a maximum of 10 times. The amount of
jitter added is returned.
Returns:
U - the Cholesky decomposition for the matrix.
jitter - the amount of jitter that was added to the matrix.
Arguments:
A - the matrix for which the Cholesky decomposition is required.
maxTries - the maximum number of times that jitter is added before
giving up (default 10).
warning - whether to give a warning for adding jitter (default is True)
See also
CHOL, PDINV, LOGDET
Copyright (c) 2005, 2006 Neil D. Lawrence
"""
jitter = 0
i = 0
while (True):
try:
# Try --- need to check A is positive definite
if jitter == 0:
jitter = abs(SP.trace(A)) / A.shape[0] * 1e-6
LC = linalg.cholesky(A, lower=True)
return LC.T, 0.0
else:
if warning:
# pdb.set_trace()
# plt.figure()
# plt.imshow(A, interpolation="nearest")
# plt.colorbar()
# plt.show()
logging.error("Adding jitter of %f in jitChol()." % jitter)
LC = linalg.cholesky(A + jitter * SP.eye(A.shape[0]),
lower=True)
return LC.T, jitter
except linalg.LinAlgError:
# Seems to have been non-positive definite.
if i < maxTries:
jitter = jitter * 10
else:
raise linalg.LinAlgError(
"Matrix non positive definite, jitter of " + str(jitter) +
" added but failed after " + str(i) + " trials.")
i += 1
return LC
def jitEigh(A, maxTries=10, warning=True):
"""
Do a Eigenvalue Decomposition with Jitter,
works as jitChol
"""
warning = True
jitter = 0
i = 0
while (True):
if jitter == 0:
jitter = abs(SP.trace(A)) / A.shape[0] * 1e-6
S, U = linalg.eigh(A)
else:
if warning:
# pdb.set_trace()
# plt.figure()
# plt.imshow(A, interpolation="nearest")
# plt.colorbar()
# plt.show()
logging.error("Adding jitter of %f in jitEigh()." % jitter)
S, U = linalg.eigh(A + jitter * SP.eye(A.shape[0]))
if S.min() > 1E-10:
return S, U
if i < maxTries:
jitter = jitter * 10
i += 1
raise linalg.LinAlgError("Matrix non positive definite, jitter of " +
str(jitter) + " added but failed after " +
str(i) + " trials.")
def _compute_precision_cholesky_full(cov):
r"""
Compute the Cholesky decomposition of the precision of the given covariance
matrix, which is expected to be a square matrix with non-zero off-diagonal
terms.
:param cov:
The given covariance matrix.
:raises scipy.linalg.LinAlgError:
If the Cholesky decomposition failed.
:returns:
The Cholesky decomposition of the precision of the covariance matrix.
"""
try:
cholesky_cov = linalg.cholesky(cov, lower=True)
except linalg.LinAlgError:
raise linalg.LinAlgError("failed to do Cholesky decomposition")
# Return the precision matrix.
D, _ = cov.shape
return linalg.solve_triangular(cholesky_cov, np.eye(D), lower=True).T
def makeAMatrix(self):
"""
Calculates the "A" matrix, that uses the existing data to find a new
component of the new phase vector.
"""
# Cholsky solve can fail - if so do brute force inversion
try:
cf = linalg.cho_factor(self.cov_mat_zz)
inv_cov_zz = linalg.cho_solve(cf, numpy.identity(self.cov_mat_zz.shape[0]))
except linalg.LinAlgError:
raise linalg.LinAlgError("Could not invert Covariance Matrix to for A and B Matrices. Try with a larger pixel scale")
self.A_mat = self.cov_mat_xz.dot(inv_cov_zz)
def _solve_cholesky(Q, z):
L, info = potrf(Q, lower=False, overwrite_a=False, clean=False)
if info > 0:
msg = "%d-th leading minor not positive definite" % info
raise la.LinAlgError(msg)
if info < 0:
msg = 'illegal value in %d-th argument of internal potrf' % -info
raise ValueError(msg)
f, info = potrs(L, z, lower=False, overwrite_b=False)
if info != 0:
msg = 'illegal value in %d-th argument of internal potrs' % -info
raise ValueError(msg)
return f
def fitEllipseDirect(points):
"""
Fits an ellipse to the given points using the method from
Fitzgibbon et al. (1991).
Parameters
----------
points : numpy.array_like
A n*2 array with the coordinates of the points to which the ellipse
will be fitted. Column 0 has the X coordinates and column 1 has the Y
coordinates.
Returns
-------
tConic : (A, B, C, D, E, F)
The conic representation touple of the fitted ellipse.
Raises
------
LinAlgError
Eigenvalue computation does not converge.
References
----------
A. Fitzgibbon, M. Pilu and R. B. Fisher. 1999. Direct least square fitting
of ellipses. IEEE Transactions on Pattern Analysis and Machine
Intelligence, vol. 21, no. 5, pp. 476-480. DOI: 10.1109/34.765658
"""
x = points[:, 0]; y = points[:, 1];
# Build design matrix
D = np.vstack((x*x, x*y, y*y, x, y, np.ones(x.shape)))
# Build scatter matrix
S = D.dot(D.T)
# Build constraint matrix
C = np.zeros((6, 6))
C[0, 2]= +2; C[1, 1]= -1; C[2, 0]= +2;
# Solve generalised eigenvalue system C*a == l*S*a
geval, gevec = linalg.eig(S, C)
# Find the eigenvector with the only pozitive eigenvalue
geval = np.real(geval)
i = np.argmax((geval>0) * np.isfinite(geval))
if not np.isfinite(geval[i]):
raise linalg.LinAlgError(
"Eigenvalue calculation failed to return a valid answer." +
"\nEigenvalues:\n" + str(geval) + '\n')
theVec = np.real(gevec[:, i])
# That vector has the parameters of the ellipse
return tuple(theVec.flatten())
def pdf(self, x):
"""
Density of the distribution at sample points.
Parameters
----------
x : :py:class:`~numpy.ndarray`
(N, p, p) values at which to determine the pdf.
Returns
-------
pdf : :py:class:`~numpy.ndarray`
(N,) densities.
"""
x = np.array(x, copy=False)
if x.ndim == 2:
x = x[np.newaxis]
elif x.ndim == 3:
pass
else:
raise ValueError('Parameter[x] must have shape (N, p, p).')
N = len(x)
if not (chk.has_shape([N, self._p, self._p])(x) and
np.allclose(x, x.conj().transpose(0, 2, 1))):
raise ValueError('Parameter[x] must be hermitian symmetric.')
if np.linalg.matrix_rank(self._V) < self._p:
raise linalg.LinAlgError('Wishart density is not defined when '
'scale matrix V is singular.')
# Determinants: real-valued since (V,X) are Hermitian.
Vs, Vl = np.linalg.slogdet(self._V)
dV = np.real(Vs * np.exp(Vl))
Xs, Xl = np.linalg.slogdet(x)
dX = np.real(Xs * np.exp(Xl))
# Trace term
A = np.linalg.solve(self._V, x)
trA = np.trace(A, axis1=1, axis2=2).real
num = (np.float_power(dX, (self._n - self._p - 1) / 2) *
np.exp(-trA / 2))
den = (np.float_power(2, self._n * self._p / 2) *
np.float_power(dV, self._n / 2) *
np.exp(special.multigammaln(self._n / 2, self._p)))
pdf = num / den
return pdf
def jit_cholesky(A, maxtries=5):
"""
Performs Jittered Cholesky Decomposition
Performs a Jittered Cholesky decomposition, adding noise to the diagonal of the matrix as needed
in order to ensure that the matrix can be inverted. Adapted from code in GPy.
On occasion, the matrix that needs to be inverted in fitting a GP is nearly singular. This arises
when the training samples are very close to one another, and can be averted by adding a noise term
to the diagonal of the matrix. This routine performs an exact Cholesky decomposition if it can
be done, and if it cannot it successively adds noise to the diagonal (starting with 1.e-6 times
the mean of the diagonal and incrementing by a factor of 10 each time) until the matrix can be
decomposed or the algorithm reaches ``maxtries`` attempts. The routine returns the lower
triangular matrix and the amount of noise necessary to stabilize the decomposition.
:param A: The matrix to be inverted as an array of shape ``(n,n)``. Must be a symmetric positive
definite matrix.
:type A: ndarray
:param maxtries: (optional) Maximum allowable number of attempts to stabilize the Cholesky
Decomposition. Must be a positive integer (default = 5)
:type maxtries: int
:returns: Lower-triangular factored matrix (shape ``(n,n)`` and the noise that was added to
the diagonal to achieve that result.
:rtype: tuple containing an ndarray and a float
"""
A = check_cholesky_inputs(A)
assert int(maxtries) > 0, "maxtries must be a positive integer"
A = np.ascontiguousarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L, 0.
else:
diagA = np.diag(A)
jitter = diagA.mean() * 1e-6
num_tries = 1
while num_tries <= maxtries and np.isfinite(jitter):
try:
L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter,
lower=True)
return L, jitter
except:
jitter *= 10
finally:
num_tries += 1
raise linalg.LinAlgError("not positive definite, even with jitter.")
return L, jitter
def solve(A, b, rtol=10**-8):
'''
Solve the matrix equation A*x=b by QR decomposition.
Parameters
----------
A : 2d ndarray
The coefficient matrix.
b : 1d ndarray
The ordinate values.
rtol : float
The relative tolerance of the solution.
Returns
-------
1d ndarray
The solution.
Raises
------
LinAlgError
When no solution exists.
'''
assert A.ndim == 2
nrow, ncol = A.shape
if nrow >= ncol:
result = np.zeros(ncol,
dtype=np.find_common_type([], [A.dtype, b.dtype]))
q, r = sl.qr(A, mode='economic', check_finite=False)
temp = q.T.dot(b)
for i, ri in enumerate(r[::-1]):
result[-1 -
i] = (temp[-1 - i] -
ri[ncol - i:].dot(result[ncol - i:])) / ri[-1 - i]
else:
temp = np.zeros(nrow,
dtype=np.find_common_type([], [A.dtype, b.dtype]))
q, r = sl.qr(dagger(A), mode='economic', check_finite=False)
for i, ri in enumerate(dagger(r)):
temp[i] = (b[i] - ri[:i].dot(temp[:i])) / ri[i]
result = q.dot(temp)
if not np.allclose(A.dot(result), b, rtol=rtol):
raise sl.LinAlgError('solve error: no solution.')
return result
def jitchol(A):
""" Do cholesky decomposition with a bit of diagonal jitter if needs be.
Aarguments:
A: a [NxN] positive definite symmetric matrix to be decomposed as
A = L.dot(L.T).
Returns:
A lower triangular matrix factor, L, also [NxN].
"""
# Try the cholesky first
try:
cholA = la.cholesky(A, lower=True)
return cholA
except la.LinAlgError as e:
pass
# Now add jitter
D = A.shape[0]
jit = 1e-13
cholA = None
di = np.diag_indices(D)
Amean = A.diagonal().mean()
while jit < 1e-3:
try:
Ajit = A.copy()
Ajit[di] += Amean * jit
cholA = la.cholesky(Ajit, lower=True)
break
except la.LinAlgError as e:
jit *= 10
if cholA is None:
raise la.LinAlgError("Too much jit! " + e.message)
return cholA
def downdate_rank_one(self, z):
"""
Rank-one down-date: compute Chol(M - x xᵀ) given L = Chol(M).
"""
# based on https://github.com/scipy/scipy/issues/8188
L = self.L
n = L.shape[0]
eps = n * np.spacing(1.) # For complex this needs modification
alpha, beta = np.empty_like(z), np.empty_like(z)
alpha[-1], beta[-1] = 1., 1.
for r in range(n):
a = z[r] / L[r, r]
alpha[r] = alpha[r - 1] - a**2
# Numerically zero or negative
if alpha[r] < eps:
# Made up err msg.
raise la.LinAlgError(
'The Cholesky factor becomes nonpositive'
'with this downdate at the step {}'.format(r))
beta[r] = np.sqrt(alpha[r])
z[r + 1:] -= a * L[r, r + 1:]
L[r, r:] *= beta[r] / beta[r - 1]
L[r, r + 1:] -= a / (beta[r] * beta[r - 1]) * z[r + 1:]
def fit(self, X, y):
"""Fit the model with X and y.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data, where n_samples is the number of samples
and n_features is the number of features.
y : array-like, shape (n_samples,)
The target values (class labels in classification, real numbers
in regression).
Returns
-------
self : object
Returns the instance itself.
"""
if sparse.issparse(X):
raise TypeError("SlicedInverseRegression does not support "
"sparse input.")
X, y = check_X_y(X,
y,
dtype=[np.float64, np.float32],
y_numeric=True,
copy=self.copy)
# handle n_directions == None
if self.n_directions is None:
n_directions = X.shape[1]
elif (not isinstance(self.n_directions, str)
and self.n_directions < 1):
raise ValueError('The number of directions `n_directions` '
'must be >= 1. Got `n_directions`={}'.format(
self.n_directions))
else:
n_directions = self.n_directions
# validate y
if is_multioutput(y):
raise TypeError("The target `y` cannot be multi-output.")
n_samples, n_features = X.shape
# Center and Whiten feature matrix using a QR decomposition
# (this is the approach used in the dr package)
if self.copy:
X = X - np.mean(X, axis=0)
else:
X -= np.mean(X, axis=0)
Q, R = linalg.qr(X, mode='economic')
Z = np.sqrt(n_samples) * Q
# sort rows of Z with respect to the target y
Z = Z[np.argsort(y), :]
# determine slices and counts
slices, counts = slice_y(y, self.n_slices)
self.n_slices_ = counts.shape[0]
# construct slice covariance matrices
M = np.zeros((n_features, n_features))
for slice_idx in range(self.n_slices_):
n_slice = counts[slice_idx]
# center the entries in this slice
Z_slice = Z[slices == slice_idx, :]
Z_slice -= np.mean(Z_slice, axis=0)
# slice covariance matrix
V_slice = np.dot(Z_slice.T, Z_slice) / n_slice
M_slice = np.eye(n_features) - V_slice
M += (n_slice / n_samples) * np.dot(M_slice, M_slice)
# eigen-decomposition of slice matrix
evals, evecs = linalg.eigh(M)
evecs = evecs[:, ::-1]
evals = evals[::-1]
try:
# TODO: internally handle zero variance features. This would not
# be a problem if we used svd, but does not match DR.
directions = linalg.solve_triangular(np.sqrt(n_samples) * R, evecs)
except (linalg.LinAlgError, TypeError):
# NOTE: The TypeError is because of a bug in the reporting of scipy
raise linalg.LinAlgError(
"Unable to back-solve R for the dimension "
"reducing directions. This is usually caused by the presents "
"of zero variance features. Try removing these features with "
"`sklearn.feature_selection.VarianceThreshold(threshold=0.)` "
"and refitting.")
# the number of directions is chosen by finding the maximum gap among
# the ordered eigenvalues.
if self.n_directions == 'auto':
n_directions = np.argmax(np.abs(np.diff(evals))) + 1
self.n_directions_ = n_directions
# normalize directions
directions = normalize(directions[:, :self.n_directions_],
norm='l2',
axis=0)
self.directions_ = directions.T
self.eigenvalues_ = evals[:self.n_directions_]
return self
def tilted(self, dQi, dri, save_fit=False):
"""Estimate the tilted distribution parameters.
This method estimates the tilted distribution parameters and calculates
the resulting site parameter updates into the given arrays. The cavity
distribution has to be calculated before this method is called, i.e. the
method cavity has to be run before this.
After calling this method the instance variables self.Mat and self.vec
hold the tilted distribution moment parameters (note however that the
covariance matrix is unnormalised and the number of samples contributing
to this matrix is stored in the instance variable self.nsamp).
Parameters
----------
dQi, dri : ndarray
Output arrays where the site parameter updates are placed.
save_fit : bool, optional
If True, the Stan fit-object is saved into the instance variable
`fit` for later use. Default is False.
Returns
-------
pos_def
True if the estimated tilted distribution covariance matrix is
positive definite. False otherwise.
"""
if self.phase != 1:
raise RuntimeError('Cavity has to be calculated before tilted.')
# FIXME: Temp fix for RandomState problem in 32-bit Python
if self.fix32bit:
self.stan_params['seed'] = self.rstate.randint(2**31 - 1)
# Sample from the model
with suppress_stdout():
time_start = timer()
fit = self.stan_model.sampling(data=self.data, **self.stan_params)
time_end = timer()
self.last_time = (time_end - time_start)
if self.verbose:
# Mean stepsize
steps = [
np.mean(p['stepsize__']) for p in fit.get_sampler_params()
]
print '\n mean stepsize: {:.4}'.format(np.mean(steps))
# Max Rhat (from all but last row in the last column)
print ' max Rhat: {:.4}'.format(
np.max(fit.summary()['summary'][:-1, -1]))
if self.init_prev:
# Store the last sample of each chain
if isinstance(self.stan_params['init'], basestring):
# No samples stored before ... initialise list of dicts
self.stan_params['init'] = get_last_fit_sample(fit)
else:
get_last_fit_sample(fit, out=self.stan_params['init'])
# Extract samples
# TODO: preallocate space for samples
samp = copy_fit_samples(fit, self.fit_pnames)
self.nsamp = samp.shape[0]
if save_fit:
# Save fit
self.fit = fit
else:
# Dereference fit here so that it can be garbage collected
fit = None
# Estimate precision matrix
try:
# Basic sample estimate
if self.prec_estim == 'sample' or self.prec_estim_skip > 0:
# Mean
mt = np.mean(samp, axis=0, out=self.vec)
# Center samples
samp -= mt
# Use QR-decomposition for obtaining Cholesky of the scatter
# matrix (only R needed, Q-less algorithm would be nice)
_, _, _, info = dgeqrf_routine(samp, overwrite_a=True)
if info:
raise linalg.LinAlgError(
"dgeqrf LAPACK routine failed with error code {}".
format(info))
# Copy the relevant part of the array into contiguous memory
np.copyto(self.Mat, samp[:self.dphi, :])
invert_normal_params(self.Mat,
mt,
out_A=dQi,
out_b=dri,
cho_form=True)
# Unbiased (for normal distr.) natural parameter estimates
unbias_k = (self.nsamp - self.dphi - 2)
dQi *= unbias_k
dri *= unbias_k
if self.prec_estim_skip > 0:
self.prec_estim_skip -= 1
# Optimal linear shrinkage estimate
elif self.prec_estim == 'olse':
# Mean
mt = np.mean(samp, axis=0, out=self.vec)
# Center samples
samp -= mt
# Sample covariance
np.dot(samp.T, samp, out=self.Mat.T)
# Normalise self.Mat into dQi
np.divide(self.Mat, self.nsamp, out=dQi)
# Estimate
olse(dQi, self.nsamp, P=self.Q, out='in-place')
np.dot(dQi, mt, out=dri)
# Graphical lasso with cross validation
elif self.prec_estim == 'glassocv':
# Mean
mt = np.mean(samp, axis=0, out=self.vec)
# Center samples
samp -= mt
# Fit
self.glassocv.fit(samp)
if self.verbose:
print ' glasso alpha: {:.4}'.format(
self.glassocv.alpha_)
np.copyto(dQi, self.glassocv.precision_.T)
# Calculate corresponding r
np.dot(dQi, mt, out=dri)
else:
raise ValueError("Invalid value for option `prec_estim`")
# Calculate the difference into the output arrays
np.subtract(dQi, self.Q, out=dQi)
np.subtract(dri, self.r, out=dri)
except linalg.LinAlgError:
# Precision estimate failed
pos_def = False
self.phase = 0
dQi.fill(0)
dri.fill(0)
if self.init_prev:
# Reset initialisation method
self.init = self.init_orig
else:
# Set return and phase flag
pos_def = True
self.phase = 2
self.iteration += 1
return pos_def
def raiseIfNan(A, error=None):
if error is None:
error = scl.LinAlgError("NaN in array")
if np.any(np.isnan(A)) or np.any(
np.isinf(A)) or np.any(abs(np.asarray(A)) > 1e30):
raise error
def conjugate_gradient(matrix,
rhs,
x0=None,
conv_tol=1e-9,
max_iter=100,
callback=None,
Pinv=None,
cg_type="polak_ribiere",
explicit_symmetrisation=IndexSymmetrisation):
"""An implementation of the conjugate gradient algorithm.
This algorithm implements the "flexible" conjugate gradient using the
Polak-Ribière formula, but allows to employ the "traditional"
Fletcher-Reeves formula as well.
It solves `matrix @ x = rhs` for `x` by minimising the residual
`matrix @ x - rhs`.
Parameters
----------
matrix
Matrix object. Should be an ADC matrix.
rhs
Right-hand side, source.
x0
Initial guess
conv_tol : float
Convergence tolerance on the l2 norm of residuals to consider
them converged.
max_iter : int
Maximum number of iterations
callback
Callback to call after each iteration
Pinv
Preconditioner to A, typically an estimate for A^{-1}
cg_type : string
Identifier to select between polak_ribiere and fletcher_reeves
explicit_symmetrisation
Explicit symmetrisation to perform during iteration to ensure
obtaining an eigenvector with matching symmetry criteria.
"""
if callback is None:
def callback(state, identifier):
pass
if explicit_symmetrisation is not None and \
isinstance(explicit_symmetrisation, type):
explicit_symmetrisation = explicit_symmetrisation(matrix)
if x0 is None:
# Start with random guess
raise NotImplementedError("Random guess is not yet implemented.")
else:
x0 = copy(x0)
if Pinv is None:
Pinv = PreconditionerIdentity()
if Pinv is not None and isinstance(Pinv, type):
Pinv = Pinv(matrix)
def is_converged(state):
state.converged = state.residual_norm < conv_tol
return state.converged
state = State()
# Initialise iterates
state.solution = x0
state.residual = evaluate(rhs - matrix @ state.solution)
state.n_applies += 1
state.residual_norm = np.sqrt(state.residual @ state.residual)
pk = zk = Pinv @ state.residual
if explicit_symmetrisation:
# TODO Not sure this is the right spot ... also this syntax is ugly
pk = explicit_symmetrisation.symmetrise(pk)
callback(state, "start")
while state.n_iter < max_iter:
state.n_iter += 1
# Update ak and iterated solution
# TODO This needs to be modified for general optimisations,
# i.e. where A is non-linear
# https://en.wikipedia.org/wiki/Nonlinear_conjugate_gradient_method
Apk = matrix @ pk
state.n_applies += 1
res_dot_zk = dot(state.residual, zk)
ak = float(res_dot_zk / dot(pk, Apk))
state.solution = evaluate(state.solution + ak * pk)
residual_old = state.residual
state.residual = evaluate(residual_old - ak * Apk)
state.residual_norm = np.sqrt(state.residual @ state.residual)
callback(state, "next_iter")
if is_converged(state):
state.converged = True
callback(state, "is_converged")
return state
if state.n_iter == max_iter:
raise la.LinAlgError("Maximum number of iterations (== " +
str(max_iter) + " reached in conjugate "
"gradient procedure.")
zk = evaluate(Pinv @ state.residual)
if explicit_symmetrisation:
# TODO Not sure this is the right spot ... also this syntax is ugly
zk = explicit_symmetrisation.symmetrise(zk)
if cg_type == "fletcher_reeves":
bk = float(dot(zk, state.residual) / res_dot_zk)
elif cg_type == "polak_ribiere":
bk = float(dot(zk, (state.residual - residual_old)) / res_dot_zk)
pk = zk + bk * pk
def fit(self, X, y):
"""Fit the model with X and y.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data, where n_samples is the number of samples
and n_features is the number of features.
y : array-like, shape (n_samples,)
The target values (class labels in classification, real numbers
in regression).
Returns
-------
self : object
Returns the instance itself.
"""
if sparse.issparse(X):
raise TypeError("SlicedInverseRegression does not support "
"sparse input.")
X, y = check_X_y(X, y, dtype=[np.float64, np.float32],
accept_sparse=['csr'],
y_numeric=True, copy=self.copy)
if self.n_directions is None:
n_directions = X.shape[1]
elif (not isinstance(self.n_directions, six.string_types) and
self.n_directions < 1):
raise ValueError('The number of directions `n_directions` '
'must be >= 1. Got `n_directions`={}'.format(
self.n_directions))
else:
n_directions = self.n_directions
if self.alpha is not None and (self.alpha <= 0 or self.alpha >= 1):
raise ValueError("The significance level `alpha` "
"must be between 0 and 1. Got `alpha`={}".format(
self.alpha))
# validate y
if is_multioutput(y):
raise TypeError("The target `y` cannot be multi-output.")
n_samples, n_features = X.shape
# Center and Whiten feature matrix using a QR decomposition
# (this is the approach used in the dr package)
if self.copy:
X = X - np.mean(X, axis=0)
else:
X -= np.mean(X, axis=0)
Q, R = linalg.qr(X, mode='economic')
Z = np.sqrt(n_samples) * Q
Z = Z[np.argsort(y), :]
# determine slice indices and counts per slice
slices, counts = slice_y(y, self.n_slices)
self.n_slices_ = counts.shape[0]
# means in each slice (sqrt factor takes care of the weighting)
Z_means = grouped_sum(Z, slices) / np.sqrt(counts.reshape(-1, 1))
M = np.dot(Z_means.T, Z_means) / n_samples
# eigen-decomposition of slice matrix
evals, evecs = linalg.eigh(M)
evecs = evecs[:, ::-1]
evals = evals[::-1]
try:
# TODO: internally handle zero variance features. This would not
# be a problem if we used svd, but does not match DR.
directions = linalg.solve_triangular(np.sqrt(n_samples) * R, evecs)
except (linalg.LinAlgError, TypeError):
# NOTE: The TypeError is because of a bug in the reporting of scipy
raise linalg.LinAlgError(
"Unable to back-solve R for the dimension "
"reducing directions. This is usually caused by the presents "
"of zero variance features. Try removing these features with "
"`sklearn.feature_selection.VarianceThreshold(threshold=0.)` "
"and refitting.")
# the number of directions is chosen by finding the maximum gap among
# the ordered eigenvalues.
if self.n_directions == 'auto':
n_directions = np.argmax(np.abs(np.diff(evals))) + 1
self.n_directions_ = n_directions
directions = normalize(
directions[:, :self.n_directions_], norm='l2', axis=0)
self.directions_ = directions.T
self.eigenvalues_ = evals[:self.n_directions_]
# Drop components in each direction using the t-ratio approach
# suggested in section 4 of Chen and Li (1998).
if self.alpha is not None:
# similar to multiple linear-regression the standard error
# estimates are proportional to the diagonals of the inverse
# covariance matrix.
precs = diagonal_precision(X, R=R)
weights = (1 - self.eigenvalues_) / (n_samples * self.eigenvalues_)
std_error = np.sqrt(weights.reshape(-1, 1) * precs)
# perform a two-sided t-test at level alpha for coefs equal to zero
# NOTE: we are not correcting for multiple tests and this is
# a very rough approximation, so do not expect the test
# to be close to the nominal level.
df = n_samples - n_features - 1
crit_val = stats.distributions.t.ppf(1 - self.alpha/2, df)
for j in range(self.n_directions_):
test_stat = np.abs(self.directions_[j, :] / std_error[j])
zero_mask = test_stat < crit_val
if np.sum(zero_mask) == n_features:
warnings.warn("Not zeroing out coefficients. All "
"coefficients are not significantly "
"different from zero.", RuntimeWarning)
else:
self.directions_[j, test_stat < crit_val] = 0.
return self
def update_single(self, j, delpi, delbeta, vvec=None):
"""
Change of EP parameters:
ep_pi[j] += delpi; ep_beta[j] += delbeta
The representation is updated accordingly. In particular, the
Cholesky factor 'lfact' is updated ('delpi'>0) or downdated
('delpi'<0). If 'keep_margs'==True, the marginal moments are
updated as well.
In 'vvec', the vector L^-1 B[j,:] can be passed. If not, it is
recomputed here.
NOTE: 'post_cov' (if given) is not updated!
"""
bfact = self.bfact
m, n = bfact.shape()
if not (isinstance(j, numbers.Integral) and j >= 0 and j < m
and isinstance(delpi, numbers.Real)
and isinstance(delbeta, numbers.Real)):
raise ValueError('J, DELPI or DELBETA wrong')
if not (vvec is None or helpers.check_vecsize(vvec, n)):
raise TypeError('VVEC wrong')
# Scratch variables. We keep them as members, to avoid having to
# allocate them in every call
try:
self.cup_c.resize(n, refcheck=False)
self.cup_s.resize(n, refcheck=False)
self.cup_wk.resize(n, refcheck=False)
self.cup_z.resize((1, n), refcheck=False)
self.us_bvec.resize(n, refcheck=False)
if self.keep_margs:
self.us_wvec.resize(m, refcheck=False)
self.us_w2vec.resize(m, refcheck=False)
except AttributeError:
self.cup_c = np.empty(n)
self.cup_s = np.empty(n)
self.cup_wk = np.empty(n)
self.cup_z = np.empty((1, n), order='F')
self.us_bvec = np.empty(n)
if self.keep_margs:
self.us_wvec = np.empty(m)
self.us_w2vec = np.empty(m)
bvec = self.us_bvec
if self.keep_margs:
wvec = self.us_wvec
w2vec = self.us_w2vec
if delpi > 0.:
# Cholesky update
tscal = np.sqrt(delpi)
bfact.T().getcol(j, bvec)
if self.keep_margs:
if vvec is None:
# Need 'vvec' below, so compute it here
vvec = sla.solve_triangular(self.lfact,
bvec,
lower=True,
trans='N')
mu = np.inner(vvec, self.cvec)
rho = np.inner(vvec, vvec)
bvec *= tscal
yscal = np.empty(1)
yscal[0] = delbeta / tscal
self.cup_z[0] = self.cvec
stat = epx.choluprk1(self.lfact, 'L', bvec, self.cup_c, self.cup_s,
self.cup_wk, self.cup_z, yscal)
if stat != 0:
raise sla.LinAlgError(
"Numerical error in 'choluprk1' (external)")
self.cvec[:] = self.cup_z.ravel()
else:
# Cholesky downdate
tscal = np.sqrt(-delpi)
if vvec is None:
bfact.T().getcol(j, bvec)
vvec = sla.solve_triangular(self.lfact,
bvec,
lower=True,
trans='N')
if self.keep_margs:
mu = np.inner(vvec, self.cvec)
rho = np.inner(vvec, vvec)
yscal = np.empty(1)
yscal[0] = -delbeta / tscal
self.cup_z[0] = self.cvec
bvec[:] = vvec
bvec *= tscal
stat = epx.choldnrk1(self.lfact, 'L', bvec, self.cup_c, self.cup_s,
self.cup_wk, self.cup_z, yscal)
if stat != 0:
raise sla.LinAlgError(
"Numerical error in 'choldnrk1' (external)")
self.cvec[:] = self.cup_z.ravel()
self.ep_pi[j] += delpi
self.ep_beta[j] += delbeta
if self.keep_margs:
# Update marginal moments
assert vvec is not None
bfact.mvm(
sla.solve_triangular(self.lfact, vvec, lower=True, trans='T'),
wvec)
tscal = 1. / (delpi * rho + 1.)
w2vec[:] = wvec
w2vec *= ((delbeta - delpi * mu) * tscal)
self.marg_means += w2vec
wvec *= wvec
wvec *= (delpi * tscal)
self.marg_vars -= wvec
def fit_toas(self, maxiter=1, threshold=False, full_cov=False):
"""Run a Generalized least-squared fitting method
If maxiter is less than one, no fitting is done, just the
chi-squared computation. In this case, you must provide the residuals
argument.
If maxiter is one or more, so fitting is actually done, the
chi-squared value returned is only approximately the chi-squared
of the improved(?) model. In fact it is the chi-squared of the
solution to the linear fitting problem, and the full non-linear
model should be evaluated and new residuals produced if an accurate
chi-squared is desired.
A first attempt is made to solve the fitting problem by Cholesky
decomposition, but if this fails singular value decomposition is
used instead. In this case singular values below threshold are removed.
full_cov determines which calculation is used. If true, the full
covariance matrix is constructed and the calculation is relatively
straightforward but the full covariance matrix may be enormous.
If false, an algorithm is used that takes advantage of the structure
of the covariance matrix, based on information provided by the noise
model. The two algorithms should give the same result to numerical
accuracy where they both can be applied.
"""
chi2 = 0
for i in range(max(maxiter, 1)):
fitp = self.get_fitparams()
fitpv = self.get_fitparams_num()
fitperrs = self.get_fitparams_uncertainty()
# Define the linear system
M, params, units, scale_by_F0 = self.get_designmatrix()
# Get residuals and TOA uncertainties in seconds
if i == 0:
self.update_resids()
residuals = self.resids.time_resids.to(u.s).value
# get any noise design matrices and weight vectors
if not full_cov:
Mn = self.model.noise_model_designmatrix(self.toas)
phi = self.model.noise_model_basis_weight(self.toas)
phiinv = np.zeros(M.shape[1])
if Mn is not None and phi is not None:
phiinv = np.concatenate((phiinv, 1 / phi))
M = np.hstack((M, Mn))
# normalize the design matrix
norm = np.sqrt(np.sum(M**2, axis=0))
ntmpar = len(fitp)
if M.shape[1] > ntmpar:
norm[ntmpar:] = 1
if np.any(norm == 0):
# Make this a LinAlgError so it looks like other bad matrixness
raise sl.LinAlgError(
"One or more of the design-matrix columns is null.")
M /= norm
# compute covariance matrices
if full_cov:
cov = self.model.covariance_matrix(self.toas)
cf = sl.cho_factor(cov)
cm = sl.cho_solve(cf, M)
mtcm = np.dot(M.T, cm)
mtcy = np.dot(cm.T, residuals)
else:
Nvec = self.model.scaled_sigma(self.toas).to(u.s).value**2
cinv = 1 / Nvec
mtcm = np.dot(M.T, cinv[:, None] * M)
mtcm += np.diag(phiinv)
mtcy = np.dot(M.T, cinv * residuals)
if maxiter > 0:
try:
c = sl.cho_factor(mtcm)
xhat = sl.cho_solve(c, mtcy)
xvar = sl.cho_solve(c, np.eye(len(mtcy)))
except sl.LinAlgError:
U, s, Vt = sl.svd(mtcm, full_matrices=False)
if threshold:
threshold_val = (np.finfo(np.longdouble).eps *
max(M.shape) * s[0])
s[s < threshold_val] = 0.0
xvar = np.dot(Vt.T / s, Vt)
xhat = np.dot(Vt.T, np.dot(U.T, mtcy) / s)
newres = residuals - np.dot(M, xhat)
# compute linearized chisq
if full_cov:
chi2 = np.dot(newres, sl.cho_solve(cf, newres))
else:
chi2 = np.dot(newres, cinv * newres) + np.dot(
xhat, phiinv * xhat)
else:
newres = residuals
if full_cov:
chi2 = np.dot(newres, sl.cho_solve(cf, newres))
else:
chi2 = np.dot(newres, cinv * newres)
return chi2
# compute absolute estimates, normalized errors, covariance matrix
dpars = xhat / norm
errs = np.sqrt(np.diag(xvar)) / norm
covmat = (xvar / norm).T / norm
self.covariance_matrix = covmat
self.correlation_matrix = (covmat / errs).T / errs
for ii, pn in enumerate(fitp.keys()):
uind = params.index(pn) # Index of designmatrix
un = 1.0 / (units[uind]) # Unit in designmatrix
if scale_by_F0:
un *= u.s
pv, dpv = fitpv[pn] * fitp[pn].units, dpars[uind] * un
fitpv[pn] = np.longdouble((pv + dpv) / fitp[pn].units)
# NOTE We need some way to use the parameter limits.
fitperrs[pn] = errs[uind]
self.minimize_func(list(fitpv.values()), *list(fitp.keys()))
# Update Uncertainties
self.set_param_uncertainties(fitperrs)
# Compute the noise realizations if possible
if not full_cov:
noise_dims = self.model.noise_model_dimensions(self.toas)
noise_resids = {}
for comp in noise_dims.keys():
p0 = noise_dims[comp][0] + ntmpar
p1 = p0 + noise_dims[comp][1]
noise_resids[comp] = np.dot(M[:, p0:p1], xhat[p0:p1]) * u.s
self.resids.noise_resids = noise_resids
return chi2
def safesvd(matrix):
"""Perform SVD with fallbacks"""
for i in range(config.svd_retry):
try:
U, s, Vd = linalg.svd(matrix, full_matrices=False)
except linalg.LinAlgError:
if i + 1 < config.svd_retry and config.linalg_verbose:
print('SVD did not converge, retrying (#%d)' % (i + 1))
else:
return U, s, Vd
global numsvd
numsvd += 1
if config.linalg_verbose:
print('SVD did not converge, manually computing SVD (#%d)' % numsvd)
try:
d0 = matrix.shape[0]
d1 = matrix.shape[1]
d = min(d0, d1)
H = np.zeros((d0 + d1, d0 + d1), dtype=complex)
H[d1:, :d1] = matrix
H[:d1, d1:] = matrix.conj().T
w, v = linalg.eigh(H)
s = np.abs(w[:d])
U = np.sqrt(2) * v[d1:, :d]
V = -np.sqrt(2) * v[:d1, :d]
# Make columns of U and of V orthonormal with Gram-Schmidt
# Start with largest singular values, so that there is most alteration
# in places which least affect result
for k in range(d):
U[:, k] -= U[:, :k].dot(U[:, :k].T.conj().dot(U[:, k]))
V[:, k] -= V[:, :k].dot(V[:, :k].T.conj().dot(V[:, k]))
U[:, k] = U[:, k] / linalg.norm(U[:, k])
V[:, k] = V[:, k] / linalg.norm(V[:, k])
return U, s, V.conj().T
except:
if config.linalg_verbose:
print('Stable method failed; using unstable method')
flip = (matrix.shape[0] < matrix.shape[1])
if flip:
matrix = matrix.T
N0, N1 = matrix.shape
# A = U.s.V^H
# A.A^H = U.s^2.U^H
U, d = linalg.eigh(matrix.dot(matrix.T.conj()))
# Return to descending order
U = U[:, :-N1 - 1:-1]
# u left singular value: u^H.A = s*v^H
V0 = U.T.conj().dot(matrix)
Vd = np.zeros(V0.shape, dtype=V0.dtype)
s = []
for i in range(N1):
v = V0[i, :]
if i:
# Gram-Schmidt
v -= Vd.T.conj().dot(V).dot(v)
norm = linalg.norm(v)
s.append(norm)
Vd[i, :] = v / norm
if flip:
U, Vd = Vd.T, U.T
if not (np.all(np.isfinite(U)) and np.all(np.isfinite(Vd)) and \
np.all(np.isfinite(s))):
raise linalg.LinAlgError('Built-in and manual SVD have failed')
return U, s, Vd
def invert_normal_params(A, b=None, out_A=None, out_b=None, cho_form=False):
"""Invert moment parameters into natural parameters or vice versa.
Switch between moment parameters (S,m) and natural parameters (Q,r) of
a multivariate normal distribution. Providing (S,m) yields (Q,r) and vice
versa.
Parameters
----------
A : ndarray
A symmetric positive-definite matrix to be inverted. Either the
covariance matrix S or the precision matrix Q.
b : {None, ndarray}, optional
The mean vector m, the natural parameter vector r, or None (default)
if `out_b` is not requested.
out_A, out_b : {None, ndarray, 'in-place'}, optional
Spesifies where the output is calculate into; None (default) indicates
that a new array is created, providing a string 'in-place' overwrites
the corresponding input array.
cho_form : bool
If True, `A` is assumed to be the upper Cholesky of the real S or Q.
Returns
-------
out_A, out_b : ndarray
The corresponding output arrays (`out_A` in F-order). If `b` was not
provided, `out_b` is None.
Raises
------
LinAlgError
If the provided array A is not positive definite.
"""
# Process parameters
if not isinstance(out_A, np.ndarray) and out_A == 'in-place':
out_A = A
elif out_A is None:
out_A = A.copy(order='F')
else:
np.copyto(out_A, A)
if not out_A.flags['FARRAY']:
# Convert from C-order to F-order by transposing (note symmetric)
out_A = out_A.T
if not out_A.flags['FARRAY'] and out_A.shape[0] > 1:
raise ValueError('Provided array A is inappropriate')
if not b is None:
if not isinstance(out_b, np.ndarray) and out_b == 'in-place':
out_b = b
elif out_b is None:
out_b = b.copy()
else:
np.copyto(out_b, b)
else:
out_b = None
# Invert
if not cho_form:
cho = linalg.cho_factor(out_A, overwrite_a=True)
else:
# Already in upper Cholesky form
cho = (out_A, False)
if not out_b is None:
linalg.cho_solve(cho, out_b, overwrite_b=True)
_, info = dpotri_routine(out_A, overwrite_c=True)
if info:
# This should never occour if cho_factor was succesful ... I think
raise linalg.LinAlgError(
"dpotri LAPACK routine failed with error code {}".format(info))
# Copy the upper triangular into the bottom
copy_triu_to_tril(out_A)
return out_A, out_b
def davidson_iterations(matrix,
state,
max_subspace,
max_iter,
n_ep,
is_converged,
which,
callback=None,
preconditioner=None,
preconditioning_method="Davidson",
debug_checks=False,
residual_min_norm=None,
explicit_symmetrisation=None):
"""
@param matrix Matrix to diagonalise
@param state DavidsonState containing the eigenvector guess
to propagate
@param max_subspace Maximal subspace size
@param max_iter Maximal numer of iterations
@param n_ep Number of eigenpairs to be computed
@param is_converged Function to test for convergence
@param callback Callback to run after each iteration
@param which Which eigenvectors to converge to.
Needs to be compatible with the selected
preconditioner.
@param preconditioner Preconditioner (type or instance)
@param preconditioning_method Precondititoning method. Valid values are
"Davidson" or "Sleijpen-van-der-Vorst"
@param debug_checks Enable some potentially costly debug checks
(loss of orthogonality in subspace etc)
@param residual_min_norm Minimal norm a residual needs to have in order
to be accepted as a new subspace vector
(defaults to 2 * len(matrix) * machine_expsilon)
@param explicit_symmetrisation Explicit symmetrisation to perform
on new subspace vectors before adding
them to the subspace.
"""
if preconditioning_method not in ["Davidson", "Sleijpen-van-der-Vorst"]:
raise ValueError("Only 'Davidson' and 'Sleijpen-van-der-Vorst' "
"are valid preconditioner methods")
if preconditioning_method == "Sleijpen-van-der-Vorst":
raise NotImplementedError("Sleijpen-van-der-Vorst preconditioning "
"not yet implemented.")
if callback is None:
def callback(state, identifier):
pass
# The problem size
n_problem = matrix.shape[1]
# The block size
n_block = len(state.subspace_vectors)
# The current subspace size
n_ss_vec = n_block
# The current subspace
SS = state.subspace_vectors
# The matrix A projected into the subspace
# as a continuous array. Only the view
# Ass[:n_ss_vec, :n_ss_vec] contains valid data.
Ass_cont = np.empty((max_subspace, max_subspace))
eps = np.finfo(float).eps
if residual_min_norm is None:
residual_min_norm = 2 * n_problem * eps
callback(state, "start")
state.timer.restart("iteration")
with state.timer.record("projection"):
# Initial application of A to the subspace
Ax = matrix @ SS
state.n_applies += n_ss_vec
while state.n_iter < max_iter:
state.n_iter += 1
assert len(SS) >= n_block
assert len(SS) <= max_subspace
# Project A onto the subspace, keeping in mind
# that the values Ass[:-n_block, :-n_block] are already valid,
# since they have been computed in the previous iterations already.
with state.timer.record("projection"):
Ass = Ass_cont[:n_ss_vec, :n_ss_vec] # Increase the work view size
for i in range(n_block):
Ass[:, -n_block + i] = Ax[-n_block + i] @ SS
Ass[-n_block:, :] = np.transpose(Ass[:, -n_block:])
# Compute the which(== largest, smallest, ...) eigenpair of Ass
# and the associated ritz vector as well as residual
with state.timer.record("rayleigh_ritz"):
if Ass.shape == (n_block, n_block):
rvals, rvecs = la.eigh(Ass) # Do a full diagonalisation
else:
# TODO Maybe play with precision a little here
# TODO Maybe use previous vectors somehow
v0 = None
rvals, rvecs = sla.eigsh(Ass, k=n_block, which=which, v0=v0)
with state.timer.record("residuals"):
# Form residuals, A * SS * v - λ * SS * v = Ax * v + SS * (-λ*v)
def form_residual(rval, rvec):
Axv = linear_combination(rvec, Ax)
Axv.add_linear_combination(-rval * rvec, SS)
return Axv
residuals = [
form_residual(rvals[i], v)
for i, v in enumerate(np.transpose(rvecs))
]
assert len(residuals) == n_block
# Update the state's eigenpairs and residuals
state.eigenvalues = select_eigenpairs(rvals, n_ep, which)
state.residuals = select_eigenpairs(residuals, n_ep, which)
state.residual_norms = np.array([r @ r for r in state.residuals])
# TODO This is misleading ... actually residual_norms contains
# the norms squared. That's also the used e.g. in adcman to
# check for convergence, so using the norm squared is fine,
# in theory ... it should just be consistent. I think it is
# better to go for the actual norm (no squared) inside the code
# TODO
# The select_eigenpairs is not that great ... better one makes a
# function which returns a mask. In that way one can have one
# form_residual function, which also returns the formed Ritz vectors
# (i.e. our approximations to the eigenpairs) and one which only
# forms the residuals ... this would save some duplicate work
# in the is_converged section *and* would allow the callback to do
# some stuff with the eigenpairs if desired.
callback(state, "next_iter")
state.timer.restart("iteration")
if is_converged(state):
# Build the eigenvectors we desire from the subspace vectors:
selected = select_eigenpairs(np.transpose(rvecs), n_ep, which)
state.eigenvectors = [linear_combination(v, SS) for v in selected]
state.converged = True
callback(state, "is_converged")
state.timer.stop("iteration")
return state
if state.n_iter == max_iter:
raise la.LinAlgError("Maximum number of iterations (== " +
str(max_iter) + " reached in davidson "
"procedure.")
if n_ss_vec + n_block > max_subspace:
callback(state, "restart")
with state.timer.record("projection"):
# The addition of the preconditioned vectors goes beyond max.
# subspace size => Collapse first, ie keep current Ritz vectors
# as new subspace
SS = [linear_combination(v, SS) for v in np.transpose(rvecs)]
state.subspace_vectors = SS
Ax = [linear_combination(v, Ax) for v in np.transpose(rvecs)]
n_ss_vec = len(SS)
# Update projection of ADC matrix A onto subspace
Ass = Ass_cont[:n_ss_vec, :n_ss_vec]
for i in range(n_ss_vec):
Ass[:, i] = Ax[i] @ SS
# continue to add residuals to space
with state.timer.record("preconditioner"):
if preconditioner:
if hasattr(preconditioner, "update_shifts"):
preconditioner.update_shifts(rvals)
preconds = preconditioner.apply(residuals)
else:
preconds = residuals
# Explicitly symmetrise the new vectors if requested
if explicit_symmetrisation:
explicit_symmetrisation.symmetrise(preconds, SS)
# Project the components of the preconditioned vectors away
# which are already contained in the subspace.
# Then add those, which have a significant norm to the subspace.
with state.timer.record("orthogonalisation"):
n_ss_added = 0
for i in range(n_block):
pvec = preconds[i]
# Project out the components of the current subspace
pvec = pvec - linear_combination(pvec @ SS, SS)
pnorm = np.sqrt(pvec @ pvec)
if pnorm > residual_min_norm:
# Extend the subspace
SS.append(pvec / pnorm)
n_ss_added += 1
n_ss_vec = len(SS)
if debug_checks:
orth = np.array([[SS[i] @ SS[j] for i in range(n_ss_vec)]
for j in range(n_ss_vec)])
orth -= np.eye(n_ss_vec)
state.subspace_orthogonality = np.max(np.abs(orth))
if state.subspace_orthogonality > n_problem * eps:
warnings.warn(
la.LinAlgWarning(
"Subspace in davidson has lost orthogonality. "
"Expect inaccurate results."))
if n_ss_added == 0:
state.converged = False
raise la.LinAlgError(
"Davidson procedure could not generate any further vectors for "
"the subpace. Iteration cannot be continued like this and will "
"be aborted without convergence. Try a different guess.")
with state.timer.record("projection"):
Ax.extend(matrix @ SS[-n_ss_added:])
state.n_applies += n_ss_added