def kpca(x, target_dim=None, kernel=gaussian_kernel, **kernel_params):
#Recall that in kpca, depending on the kernel, each point can span a new direction in feature space...
num_examples, num_features = shape(x)
if not target_dim:
target_dim = num_features #...we usually do not want that many
#Kernel matrix
K = kernel_matrix(x, kernel=kernel, **kernel_params)
#Center the data in feature space: K' = K -1nK -K1n +1nK1n
#Some algebra can optimize this both in time and space, think...
onen = ones((num_examples, num_examples)) / num_examples
onenK = dot(onen, K)
K = K - onenK - onenK.T + dot(onenK, onen)
#Eigendecomposition
eigenvalues, eigenvectors = eig(K)
#Selection of the PCs
eigenvalues, eigenvectors, inertias = select_top(eigenvalues, eigenvectors, target_dim)
#Transform
x = dot(diag(sqrt(eigenvalues)), eigenvectors.T).T
return x, K, eigenvectors, eigenvalues, inertias
def nystrom_experiment(X, m0, mmax, kernel, dataset):
"""
Incremental calculation of the Nyström approximation to the kernel matrix.
For each data point the difference in Frobenius norm between the
approximation and the full kernel matrix is plotted.
Parameters
----------
X : numpy.ndarray, 2d
Data matrix
m0 : int
Initial size of Nyström subset
mmax : int
Maximum size of Nyström subset
kernel : callable
Kernel function
dataset : str
Either 'magic' or 'yeast'
"""
print("\nIncremental Nyström approximation")
inc = IncrKPCA(X, m0, mmax, kernel=kernel, nystrom=True)
idx = inc.get_idx_array()
n = X.shape[0]
K = kernel_matrix(X, kernel, range(n), range(n))
fnorms = []
for i, L, U, L_nys, U_nys in inc:
K_tilde = dot(U_nys, dot(diag(L_nys), U_nys.T))
fnorm = np.sqrt(np.sum(np.sum(np.power(K - K_tilde, 2))))
fnorms.append(fnorm)
plotting(range(m0, m0 + len(fnorms)), fnorms, dataset, "m",
"Frobenius norm")
def update_K_nm(self):
"""
Update K_nm for one iteration by adding another column
"""
i = self.i
K_ni = kernel_matrix(self.X, self.kernel, range(self.n), [self.idx[i]])
self.K_nm = np.c_[self.K_nm, K_ni]
def create_update_terms(X, cols, col, kernel):
"""
Create the terms supplied to eigenvalue update algorithm
Parameters
----------
X : np.ndarray, 2d
Data matrix
cols : np.ndarray, 1d
Indices of columns to create the kernel matrix
col : float
The additional column index
Returns
-------
Parameters supplied to update algorithm for
eigendecomposition
"""
k1 = kernel_matrix(X, kernel, cols, [col])
k = copy(k1[-1][0])
k1[-1] = k / 2
k0 = deepcopy(k1) # numpy pass by reference
k0[-1] = k / 4
sigma = 4 / k
return sigma, k1, k0
def __init__(self,
X,
m0,
mmax=None,
kernel=kernel_error,
adjust=False,
nystrom=False,
maxiter=500):
# Setup default arguments
n = X.shape[0]
self.X = X
self.i = m0
self.m0 = m0
self.j = 0
self.n = n
self.maxiter = maxiter
if mmax is None:
mmax = n
self.mmax = min(mmax, n)
self.idx = np.random.permutation(n)
self.kernel = kernel
self.adjust = adjust
self.nystrom = nystrom
# Initial eigensystem
cols = self.idx[:m0]
K_mm = kernel_matrix(X, kernel, cols, cols)
if self.adjust:
self.L, self.U, self.capsig, self.K1 = init_vars(K_mm)
else:
self.L, self.U = linalg.eigh(K_mm)
if self.nystrom:
self.K_nm = kernel_matrix(X, kernel, range(n), cols)
def test_nystrom_approx():
datasize = 100
fraction = 0.1
X = get_magic_data()
X = X[:datasize]
cols = range(int(fraction * datasize))
all_cols = range(datasize)
K_mm, K_nm = kernel_matrices(X, rbf, cols)
K = kernel_matrix(X, rbf, all_cols, all_cols)
L, U = linalg.eigh(K_mm)
L_nys, U_nys = nystrom_approximation(L, U, K_nm)
K_nys = dot(U_nys, dot(diag(L_nys), U_nys.T))
# F norm of difference
fnorm = np.sqrt(np.sum(np.sum(np.power(K - K_nys, 2))))
assert_less(fnorm / datasize, datasize)
def kpca_cv_experiment(X, kernel, dataset, n_iter, kernel_label):
"""
"""
print("\nCross-validation of kernel PCA\n------------------------------")
K = kernel_matrix(X, kernel)
n = K.shape[0]
print("Number of data points: {}".format(n))
print("Kernel: {}".format(kernel_label))
pc, errs = kpca_cv(K, n_iter)
print("Selected PC: {}".format(pc))
err_mean = errs.mean(0)
title = dataset + " " + kernel_label
plotting(np.arange(n_iter - 1) + 1, err_mean, title, "k", "Mean error")
def predict(weights, train_xs, test_xs, test_ys, kernel_func, dim, kernel=None):
"""
Predict classes of test set using multi-class kernel perceptron algorithm
:param weights: weights fit to train set
:param train_xs: train set
:param test_xs: test set
:param test_ys: test classes
:param kernel_func: kernel function
:param dim: dimension used for kernel function
:param kernel: kernel matrix
:return: test error
"""
K = kernel if kernel is not None else kernel_matrix(test_xs, train_xs, dim, kernel_func)
num_correct = 0
w = (weights @ K.T).T
for index, x in enumerate(test_xs):
predicted = np.argmax(w[index, :])
correct = test_ys[index]
if predicted == correct:
num_correct += 1
return 1 - num_correct / len(test_xs)
def incremental_experiment(X, m0, mmax, kernel, dataset, adjust=False):
"""
Experiment for the incremental kernel pca algorithm. For each additional
data point the difference in Frobenius norm between incremental and batch
calculation is plotted (termed drift).
Parameters
----------
X : numpy.ndarray, 2d
Data matrix
m0 : int
Initial size of kernel matrix
mmax : int
Maximum size of kernel matrix
kernel : callable
Kernel function
dataset : str
Either 'magic' or 'yeast'
adjust : bool
Whether to adjust the mean
"""
print("\nIncremental kernel PCA")
inc = IncrKPCA(X, m0, mmax, adjust=adjust, kernel=kernel)
fnorms = []
for i, L, U in inc:
idx = inc.get_idx_array()
K = kernel_matrix(X, kernel, idx[:i + 1], idx[:i + 1])
if adjust:
K = adjust_K(K)
K_tilde = dot(U, dot(diag(L), U.T))
fnorm = np.sqrt(np.sum(np.sum(np.power(K - K_tilde, 2))))
fnorms.append(fnorm)
plotting(
np.arange(len(fnorms)) + m0, fnorms, dataset, "m", "Frobenius norm")
def kpca(x, target_dim=None, kernel=gaussian_kernel, shuffle=None, **kernel_params):
#Recall that in kpca, depending on the kernel, each point can span a new direction in feature space...
num_examples, num_features = shape(x)
if not target_dim:
target_dim = num_features #...we usually do not want that many
#Compute the kernel matrix
K = kernel_matrix(x, kernel=kernel, **kernel_params)
if shuffle:
K = shuffle_matrix(K, shuffle)
#Center the data in feature space: K' = K -1nK -K1n +1nK1n
K = center_kernel_matrix(K)
#Eigendecomposition
eigenvalues, eigenvectors = eig(K)
#Selection of the PCs
eigenvalues, eigenvectors, inertias = select_top(eigenvalues, eigenvectors, target_dim)
#Transform
x = dot(diag(sqrt(eigenvalues)), eigenvectors.T).T
return x, K, eigenvectors, eigenvalues, inertias
def update_eig_adjust(self):
"""
Update the kernel PCA solution including adjustment of the mean.
"""
i = self.i
col = self.idx[i]
cols = self.idx[:i + 1]
k = kernel_matrix(self.X, self.kernel, cols, [col]) # OK
a = k[:-1, :]
a_sum = np.sum(a)
k_sum = np.sum(k)
capsig2 = self.capsig + 2 * a_sum + k[-1, 0]
C = -self.capsig / i**2 + capsig2 / (i + 1)**2
u = self.K1 / (i * (i + 1)) - a / (i + 1) + 0.5 * C * ones((i, 1))
u1 = 1 + u
u2 = 1 - u
sigma_u = 0.5
K1 = np.r_[self.K1 + a, [[k_sum]]]
capsig = capsig2
v = k - (ones((i + 1, 1)) * k_sum + K1 - capsig / (i + 1)) / (i + 1)
v1 = deepcopy(v)
v2 = deepcopy(v)
v0 = copy(v[-1, 0])
v1[-1, 0] = v0 / 2
v2[-1, 0] = v0 / 4
sigma_k = 4 / v0
# Apply rank one updates
L, U = update_eigensystem(self.L, self.U, u1, sigma_u)
if isinstance(L, np.ndarray):
L, U = update_eigensystem(L, U, u2, -sigma_u)
if isinstance(L, np.ndarray):
L, U = expand_eigensystem(L, U, v0 / 4)
# Ordering
idx = np.argsort(L)
L = L[idx]
U = U[:, idx]
U = U[idx, :]
v1 = v1[idx, :]
v2 = v2[idx, :]
L, U = update_eigensystem(L, U, v1, sigma_k)
if isinstance(L, np.ndarray):
L, U = update_eigensystem(L, U, v2, -sigma_k)
if isinstance(L, np.ndarray):
#f self.nystrom:
# self.update_K_nm()
K1 = K1[idx, :]
self.idx[:i + 1] = self.idx[:i + 1][idx] # Reorder index
if self.nystrom:
self.K_nm = self.K_nm[:, idx] # Reorder columns
self.i, self.L, self.U, self.K1 = i + 1, L, U, K1
self.capsig = capsig
rc = 0
else: # Ignore data example
self.idx[i:-1] = self.idx[i + 1:]
self.idx = self.idx[:-1]
rc = 1
return rc
