def cholup(R, x, sgn):
u = x.copy()
if sgn == '+':
cholupdate(R, u)
elif sgn == '-':
choldowndate(R, u)
return R
def _removeOne(self, x):
assert x.ndim <= 1
assert x.size == self._dim
self._count -= 1
self._n -= 1
self._k -= 1
self._mu = self._mu - (x - self._mu) / self._k
choldate.choldowndate(self._U, np.copy(x))
def U(self):
if 'U' in self._cache:
return self._cache['U']
Utemp = np.copy((self._U))
choldate.choldowndate(Utemp, sqrt(self._k) * self._mu)
trueU = Utemp * sqrt((1 + (1 / self._k)) / (self.nu))
# update cache
self._cache['U'] = trueU
return trueU
def test_downdate(self):
V = numpy.dot(self.X.transpose(),self.X)
R = numpy.linalg.cholesky(V).transpose()
u = numpy.random.normal(size=R.shape[0])
V1 = V + numpy.outer(u,u)
R1 = numpy.linalg.cholesky(V1).transpose()
R_ = R1.copy()
u_ = u.copy()
choldowndate(R_,u_)
self.assertAlmostEqual(numpy.max((numpy.abs(R_) - numpy.abs(R))**2),0)
def downdate_cluster_params(mean, cov_chol, data_point, n_cluster):
kappa_0 = cgs_utils.init_kappa_0()
new_mean = (mean * (kappa_0 + n_cluster) -
data_point) / (kappa_0 + n_cluster - 1)
u_vec = np.sqrt(
(kappa_0 + n_cluster) /
(kappa_0 + n_cluster - 1)) * (data_point - mean).astype(np.float64)
# overriding old covariance matrix
current_cov_chol = cov_chol.astype(np.float64).T
choldate.choldowndate(current_cov_chol, u_vec.copy())
return new_mean.astype(np.float32), current_cov_chol.T.astype(
np.float32)
def UpdateFactor(factor, index, Z, theta_rbf, theta_band, K, X_var_d):
# make sure we use upper matrix
assert factor[1] == False
# Z_new = np.array(Z)
# Z_new[index, :] = pars
K_new = kernelRBF(Z, theta_rbf, theta_band)
K_new[np.diag_indices_from(K_new)] += X_var_d
u = K_new[index, :] - K[index, :]
u[index] = 1.
cholupdate(factor[0], u.copy())
u[index] = 0.
choldowndate(factor[0], u)
w = np.zeros_like(u)
w[index] = 1.
choldowndate(factor[0], w)
return factor, K_new
def step_simplex(self, state, randomization, logpdf):
self.total_l1_part += 1
lam = self.lagrange
data, opt_vars = state
simplex, cube = opt_vars
if self.lagrange is None:
raise NotImplementedError(
"The bound form has not been implemented")
nactive = simplex.shape[0]
stepsize = 1.5 / np.sqrt(nactive)
rand = randomization
random_sample = rand.rvs(size=nactive)
step = np.dot(self.chol_adapt, random_sample)
proposal = np.fabs(simplex + step)
log_ratio = (logpdf((data, (proposal, cube))) - logpdf(state))
# update cholesky factor
alpha = np.minimum(np.exp(log_ratio), 1)
target = 2.4 / np.sqrt(nactive)
multiplier = ((self.total_l1_part + 1)**(-0.8) *
(np.exp(log_ratio) - target))
rank_one = np.sqrt(
np.fabs(multiplier)) * step / np.linalg.norm(random_sample)
if multiplier > 0:
cholupdate(self.chol_adapt, rank_one) # update done in place
else:
choldowndate(self.chol_adapt, rank_one) # update done in place
if np.log(np.random.uniform()) < log_ratio:
simplex = proposal
self.accept_l1_part += 1
return simplex
def step_simplex(self, state, randomization, logpdf):
self.total_l1_part += 1
lam = self.lagrange
data, opt_vars = state
simplex, cube = opt_vars
if self.lagrange is None:
raise NotImplementedError("The bound form has not been implemented")
nactive = simplex.shape[0]
stepsize = 1.5/np.sqrt(nactive)
rand = randomization
random_sample = rand.rvs(size=nactive)
step = np.dot(self.chol_adapt, random_sample)
proposal = np.fabs(simplex + step)
log_ratio = (logpdf((data, (proposal, cube))) -
logpdf(state))
# update cholesky factor
alpha = np.minimum(np.exp(log_ratio), 1)
target = 2.4 / np.sqrt(nactive)
multiplier = ((self.total_l1_part+1)**(-0.8) *
(np.exp(log_ratio) - target))
rank_one = np.sqrt(np.fabs(multiplier)) * step / np.linalg.norm(random_sample)
if multiplier > 0:
cholupdate(self.chol_adapt, rank_one) # update done in place
else:
choldowndate(self.chol_adapt, rank_one) # update done in place
if np.log(np.random.uniform()) < log_ratio:
simplex = proposal
self.accept_l1_part += 1
return simplex
def chol_rank1_downdate(L, x):
choldowndate(L.T, x.copy())
def MaskSolve(A, b, w=5, progress=True, niter=None):
'''
Finds the solution `x` to the linear problem
A x = b
for all contiguous `w`-sized masks applied to
the rows and columns of `A` and to the entries
of `b`.
Returns an array `X` of shape `(N - w + 1, N - w)`,
where the `nth` row is the solution to the equation
A[![n,n+w)] x = b[![n,n+w)]
where ![n,n+w) indicates that indices in the range
[n,n+w) have been masked.
'''
# Ensure we have choldate installed
if cholupdate is None:
log.info("Running the slow version of `MaskSolve`.")
log.info("Install the `choldate` package for better performance.")
log.info("https://github.com/rodluger/choldate")
return MaskSolveSlow(A, b, w=w, progress=progress, niter=niter)
# Number of data points
N = b.shape[0]
# How many iterations? Default is to go through
# the entire dataset
if niter is None:
niter = N - w + 1
# Our result matrix
X = np.empty((niter, N - w))
# Solve the first two steps explicitly.
for n in range(2):
mask = np.arange(n, w + n)
A_ = np.delete(np.delete(A, mask, axis=0), mask, axis=1)
b_ = np.delete(b, mask)
U = cholesky(A_)
X[n] = cho_solve((U, False), b_)
# Iterate!
for n in prange(1, niter - 1):
# Update the data vector.
b_[n] = b[n]
# Remove a row.
S33 = U[n + 1:, n + 1:]
S23 = U[n, n + 1:]
cholupdate(S33, S23)
# Add a row.
A12 = A[:n, n]
A22 = A[n, n]
A23 = A[n, n + w + 1:]
S11 = U[:n, :n]
S12 = solve_triangular(S11.T, A12, lower=True,
check_finite=False, trans=0, overwrite_b=True)
S22 = np.sqrt(A22 - np.dot(S12.T, S12))
S13 = U[:n, n + 1:]
S23 = (A23 - np.dot(S12.T, S13)) / S22
choldowndate(S33, np.array(S23))
U[:n, n] = S12
U[n, n] = S22
U[n, n + 1:] = S23
U[n + 1:, n + 1:] = S33
# Now we can solve our linear equation
X[n + 1] = cho_solve((U, False), b_)
# Return the matrix
return X
Created on Feb 15, 2013 @author: jasonrudy ''' from choldate import cholupdate, choldowndate import numpy #Create a random positive definite matrix, V numpy.random.seed(1) X = numpy.random.normal(size=(100,10)) V = numpy.dot(X.transpose(),X) #Calculate the upper Cholesky factor, R R = numpy.linalg.cholesky(V).transpose() #Create a random update vector, u u = numpy.random.normal(size=R.shape[0]) #Calculate the updated positive definite matrix, V1, and its Cholesky factor, R1 V1 = V + numpy.outer(u,u) R1 = numpy.linalg.cholesky(V1).transpose() #The following is equivalent to the above R1_ = R.copy() cholupdate(R1_,u.copy()) assert(numpy.all((R1 - R1_)**2 < 1e-16)) #And downdating is the inverse of updating R_ = R1.copy() choldowndate(R_,u.copy()) assert(numpy.all((R - R_)**2 < 1e-16))
def _solve_maxent_sdp_cd(
Sigma,
solve_sdp,
tol=1e-5,
verbose=False,
num_iter=50,
converge_tol=1e-4,
choldate_warning=True,
mu=0.9,
lambd=0.5,
):
"""
This function is internally used to compute the S-matrices
used to generate maximum entropy and SDP knockoffs. Users
should not call this function---they should call ``solve_maxent``
or ``solve_sdp`` directly.
Parameters
----------
Sigma : np.ndarray
``(p, p)``-shaped covariance matrix of X
tol : float
Minimum permissible eigenvalue of 2Sigma - S and S.
verbose : bool
If True, prints updates during optimization.
num_iter : int
The number of coordinate descent iterations. Defaults to 50.
converge_tol : float
A parameter specifying the criteria for convergence.
choldate_warning : bool
If True, will warn the user if choldate is not installed.
Defaults to True
solve_sdp : bool
If True, will solve SDP. Otherwise, will solve maxent formulation.
lambd : float
Initial barrier constant
mu : float
Barrier decay constant
Returns
-------
S : np.ndarray
``(p, p)``-shaped (block) diagonal matrix used to generate knockoffs
"""
# Warning if choldate not available
if not CHOLDATE_AVAILABLE and choldate_warning:
warnings.warn(constants.CHOLDATE_WARNING)
# Initial constants
time0 = time.time()
V = Sigma # Shorthand prevents lines from spilling over
p = V.shape[0]
inds = np.arange(p)
loss = np.inf
# Initialize values
decayed_improvement = 1
mineig = np.linalg.eigh(V)[0].min()
if solve_sdp:
S = 0.01 * mineig * np.eye(p)
else:
S = mineig * np.eye(p)
L = np.linalg.cholesky(2 * V - S)
lambd = min(2 * mineig, lambd)
# Loss function
if solve_sdp:
loss_fn = lambda V, S: S.shape[0] - np.diag(S).sum()
else:
loss_fn = maxent_loss
for i in range(num_iter):
np.random.shuffle(inds)
for j in inds:
diff = 2 * V - S
# Solve cholesky equation
tildey = 2 * V[j].copy()
tildey[j] = 0
x = sp.linalg.solve_triangular(a=L, b=tildey, lower=True)
# Use cholesky eq to get new update
zeta = diff[j, j]
x22 = np.power(x, 2).sum()
qinvterm = zeta * x22 / (zeta + x22)
# Inverse of Qj using SWM formula
if solve_sdp:
sjstar = max(min(1, 2 * V[j, j] - qinvterm - lambd), 0)
else:
sjstar = (2 * V[j, j] - qinvterm) / 2
# Rank one update for cholesky
delta = S[j, j] - sjstar
x = np.zeros(p)
x[j] = np.sqrt(np.abs(delta))
if delta > 0:
if CHOLDATE_AVAILABLE:
choldate.cholupdate(L.T, x)
else:
cholupdate(L.T, x, add=False)
else:
if CHOLDATE_AVAILABLE:
choldate.choldowndate(L.T, x)
else:
cholupdate(L.T, x, add=True)
# Set new value for S
S[j, j] = sjstar
# Check for convergence
prev_loss = loss
loss = loss_fn(V, S)
if i != 0:
loss_diff = prev_loss - loss
if solve_sdp:
loss_diff = max(loss_diff, lambd)
decayed_improvement = decayed_improvement / 10 + 9 * (
loss_diff) / 10
if verbose:
print(
f"After iter {i} at time {np.around(time.time() - time0,3)}, loss={loss}, decayed_improvement={decayed_improvement}"
)
if decayed_improvement < converge_tol:
if verbose:
print(f"Converged after iteration {i} with loss={loss}")
break
# Update barrier parameter if solving SDP
if solve_sdp:
lambd = mu * lambd
# Ensure validity of solution
S = utilities.shift_until_PSD(S, tol=tol)
S, _ = utilities.scale_until_PSD(V, S, tol=tol, num_iter=10)
return S
def _solve_mvr_ungrouped(
Sigma,
tol=1e-5,
verbose=False,
num_iter=50,
smoothing=0,
rej_rate=0,
converge_tol=1e-2,
choldate_warning=True,
):
"""
Computes S-matrix used to generate minimum variance-based
reconstructability knockoffs using coordinate descent.
Parameters
----------
Sigma : np.ndarray
``(p, p)``-shaped covariance matrix of X
tol : float
Minimum permissible eigenvalue of 2Sigma - S and S.
verbose : bool
If True, prints updates during optimization.
num_iter : int
The number of coordinate descent iterations. Defaults to 50.
smoothing : float
Add ``smoothing`` to all eigenvalues of the feature-knockoff
precision matrix before inverting to avoid numerical
instability. Defaults to 0.
converge_tol : float
A parameter specifying the criteria for convergence.
choldate_warning : bool
If True, will warn the user if choldate is not installed.
Defaults to True.
Returns
-------
S : np.ndarray
``(p, p)``-shaped (block) diagonal matrix used to generate knockoffs
"""
# Warning if choldate not available
if not CHOLDATE_AVAILABLE and choldate_warning:
warnings.warn(constants.CHOLDATE_WARNING)
# Initial constants
time0 = time.time()
V = Sigma # Shorthand prevents lines from spilling over
p = V.shape[0]
inds = np.arange(p)
loss = np.inf
acc_rate = 1 - rej_rate
# Initialize values
decayed_improvement = 10
min_eig = np.linalg.eigh(V)[0].min()
S = min_eig * np.eye(p)
L = np.linalg.cholesky(2 * V - S + smoothing * np.eye(p))
for i in range(num_iter):
np.random.shuffle(inds)
for j in inds:
# 1. Compute coefficients cn and cd
ej = np.zeros(p) # jth basis element
ej[j] = 1
# 1a. Compute cd
vd = sp.linalg.solve_triangular(a=L, b=ej, lower=True)
cd = np.power(vd, 2).sum()
# 1b. Compute vn
vn = sp.linalg.solve_triangular(a=L.T, b=vd, lower=False)
cn = -1 * np.power(vn, 2).sum()
# 2. Construct/solve quadratic equation
delta = _solve_mvr_quadratic(
cn=cn,
cd=cd,
sj=S[j, j],
min_eig=min_eig,
i=i,
smoothing=smoothing,
acc_rate=acc_rate,
)
# 3. Update S and L
x = np.zeros(p)
x[j] = np.sqrt(np.abs(delta))
if delta > 0:
if CHOLDATE_AVAILABLE:
choldate.choldowndate(L.T, x)
else:
cholupdate(L.T, x, add=False)
else:
if CHOLDATE_AVAILABLE:
choldate.cholupdate(L.T, x)
else:
cholupdate(L.T, x, add=True)
# Set new value for S
S[j, j] += delta
# Check for convergence
prev_loss = loss
loss = mvr_loss(V, acc_rate * S, smoothing=smoothing)
if i != 0:
decayed_improvement = decayed_improvement / 10 + 9 * (prev_loss -
loss) / 10
if verbose:
print(
f"After iter {i} at time {np.around(time.time() - time0,3)}, loss={loss}, decayed_improvement={decayed_improvement}"
)
if decayed_improvement < converge_tol:
if verbose:
print(f"Converged after iteration {i} with loss={loss}")
break
return S
Created on Feb 15, 2013 @author: jasonrudy ''' from choldate import cholupdate, choldowndate import numpy #Create a random positive definite matrix, V numpy.random.seed(1) X = numpy.random.normal(size=(100, 10)) V = numpy.dot(X.transpose(), X) #Calculate the upper Cholesky factor, R R = numpy.linalg.cholesky(V).transpose() #Create a random update vector, u u = numpy.random.normal(size=R.shape[0]) #Calculate the updated positive definite matrix, V1, and its Cholesky factor, R1 V1 = V + numpy.outer(u, u) R1 = numpy.linalg.cholesky(V1).transpose() #The following is equivalent to the above R1_ = R.copy() cholupdate(R1_, u.copy()) assert (numpy.all((R1 - R1_)**2 < 1e-16)) #And downdating is the inverse of updating R_ = R1.copy() choldowndate(R_, u.copy()) assert (numpy.all((R - R_)**2 < 1e-16))
def solve_mvr(Sigma,
tol=1e-5,
verbose=False,
num_iter=10,
smoothing=0,
rej_rate=0,
converge_tol=1):
"""
Uses a coordinate-descent algorithm to find the solution to the smoothed
MVR problem.
:param Sigma: p x p covariance matrix
:param tol: Minimum eigenvalue of 2Sigma - S and S
:param num_iter: Number of coordinate descent iterations
:param rej_rate: Expected proportion of rejections for knockoffs under the
metropolized knockoff sampling framework.
:param verbose: if true, will give progress reports
:param smoothing: computes smoothed mvr loss
"""
# Initial constants
time0 = time.time()
V = Sigma # I'm too lazy to write Sigma out over and over
p = V.shape[0]
inds = np.arange(p)
loss = np.inf
acc_rate = 1 - rej_rate
# Takes a bit longer for rej_rate adjusted to converge
if acc_rate < 1:
converge_tol = 1e-2
# Initialize values
decayed_improvement = 10
min_eig = np.linalg.eigh(V)[0].min()
S = min_eig * np.eye(p)
L = np.linalg.cholesky(2 * V - S + smoothing * np.eye(p))
for i in range(num_iter):
np.random.shuffle(inds)
for j in inds:
# 1. Compute coefficients cn and cd
ej = np.zeros(p) # jth basis element
ej[j] = 1
# 1a. Compute cd
vd = sp.linalg.solve_triangular(a=L, b=ej, lower=True)
cd = np.power(vd, 2).sum()
# 1b. Compute vn
vn = sp.linalg.solve_triangular(a=L.T, b=vd, lower=False)
cn = -1 * np.power(vn, 2).sum()
# 2. Construct quadratic equation
# We want to minimize 1/(sj + delta) - (delta * cn)/(1 - delta * cd)
coef2 = -1 * cn - np.power(cd, 2)
coef1 = 2 * (-1 * cn * (S[j, j] + smoothing) + cd)
coef0 = -1 * cn * (S[j, j] + smoothing)**2 - 1
orig_options = np.roots(np.array([coef2, coef1, coef0]))
# 3. Eliminate complex solutions
options = np.array(
[delta for delta in orig_options if np.imag(delta) == 0])
# Eliminate solutions which violate PSD-ness
upper_bound = 1 / cd
lower_bound = -1 * S[j, j]
options = np.array([
delta for delta in options
if delta < upper_bound and delta > lower_bound
])
if options.shape[0] == 0:
raise RuntimeError(
f"All quadratic solutions ({orig_options}) were infeasible or imaginary"
)
# 4. If multiple solutions left (unlikely), pick the smaller one
losses = 1 / (S[j, j] + options) - (options * cn) / (1 -
options * cd)
if losses[0] == losses.min():
delta = options[0]
else:
delta = options[1]
# 5. Account for rejections
if acc_rate < 1:
extra_space = min(min_eig, 0.02) / (
i + 2) # Helps deal with coord desc
opt_postrej_value = S[j, j] + delta
opt_prerej_value = opt_postrej_value / (acc_rate)
opt_prerej_value = min(S[j, j] + upper_bound - extra_space,
max(opt_prerej_value, extra_space))
delta = opt_prerej_value - S[j, j]
# Update S and L
x = np.zeros(p)
x[j] = np.sqrt(np.abs(delta))
if delta > 0:
choldate.choldowndate(L.T, x)
else:
choldate.cholupdate(L.T, x)
# Set new value for S
S[j, j] += delta
# Check for convergence
prev_loss = loss
loss = mvr_loss(V, acc_rate * S, smoothing=smoothing)
if i != 0:
decayed_improvement = decayed_improvement / 10 + 9 * (prev_loss -
loss) / 10
if verbose:
print(
f"After iter {i} at time {np.around(time.time() - time0,3)}, loss={loss}, decayed_improvement={decayed_improvement}"
)
if decayed_improvement < converge_tol:
if verbose:
print(f"Converged after iteration {i} with loss={loss}")
break
# Ensure validity of solution
S = utilities.shift_until_PSD(S, tol=tol)
S, _ = utilities.scale_until_PSD(V, S, tol=tol, num_iter=10)
return S
def solve_maxent(Sigma,
tol=1e-5,
verbose=False,
num_iter=10,
smoothing=0,
converge_tol=1e-4):
"""
Uses a coordinate-descent algorithm to find the solution to the smoothed
maximum entropy problem.
:param Sigma: p x p covariance matrix
:param tol: Minimum eigenvalue of 2Sigma - S and S
:param num_iter: Number of coordinate descent iterations
:param verbose: if true, will give progress reports
:param smoothing: computes smoothed maxent loss
"""
if smoothing > 0:
raise NotImplementedError(f"Smoothing is not implemented yet")
# Initial constants
time0 = time.time()
V = Sigma # I'm too lazy to write Sigma out over and over
p = V.shape[0]
inds = np.arange(p)
loss = np.inf
# Initialize values
decayed_improvement = 1
S = np.linalg.eigh(V)[0].min() * np.eye(p)
L = np.linalg.cholesky(2 * V - S)
for i in range(num_iter):
np.random.shuffle(inds)
for j in inds:
diff = 2 * V - S
# Solve cholesky equation
tildey = 2 * V[j].copy()
tildey[j] = 0
x = sp.linalg.solve_triangular(a=L, b=tildey, lower=True)
# Use cholesky eq to get new update
zeta = diff[j, j]
x22 = np.power(x, 2).sum()
qinvterm = zeta * x22 / (zeta + x22)
# Inverse of Qj using SWM formula
sjstar = (2 * V[j, j] - qinvterm) / 2
# Rank one update for cholesky
delta = S[j, j] - sjstar
x = np.zeros(p)
x[j] = np.sqrt(np.abs(delta))
if delta > 0:
choldate.cholupdate(L.T, x)
else:
choldate.choldowndate(L.T, x)
# Set new value for S
S[j, j] = sjstar
# Check for convergence
prev_loss = loss
loss = maxent_loss(V, S, smoothing=smoothing)
if i != 0:
decayed_improvement = decayed_improvement / 10 + 9 * (prev_loss -
loss) / 10
if verbose:
print(
f"After iter {i} at time {np.around(time.time() - time0,3)}, loss={loss}, decayed_improvement={decayed_improvement}"
)
if decayed_improvement < converge_tol:
if verbose:
print(f"Converged after iteration {i} with loss={loss}")
break
# Ensure validity of solution
S = utilities.shift_until_PSD(S, tol=tol)
S, _ = utilities.scale_until_PSD(V, S, tol=tol, num_iter=10)
return S
