def chooseKernel(data, kerneltype='euclidean'):
r"""Kernalize data (uses sklearn)
Parameters
==========
data : array of shape (n_individuals, n_dimensions)
Data matrix.
kerneltype : {'euclidean', 'cosine', 'laplacian', 'polynomial_kernel', 'jaccard'}, optional
Kernel type.
Returns
=======
array of shape (n_individuals, n_individuals)
"""
if kerneltype == 'euclidean':
K = np.divide(1, (1+pairwise_distances(data, metric="euclidean")))
elif kerneltype == 'cosine':
K = (pairwise.cosine_kernel(data))
elif kerneltype == 'laplacian':
K = (pairwise.laplacian_kernel(data))
elif kerneltype == 'linear':
K = (pairwise.linear_kernel(data))
elif kerneltype == 'polynomial_kernel':
K = (pairwise.polynomial_kernel(data))
elif kerneltype == 'jaccard':
K = 1-distance.cdist(data, data, metric='jaccard')
scaler = KernelCenterer().fit(K)
return(scaler.transform(K))
def test_kernelcenterer_vs_sklearn():
# Compare msmbuilder.preprocessing.KernelCenterer
# with sklearn.preprocessing.KernelCenterer
kernelcentererr = KernelCentererR()
kernelcentererr.fit(np.concatenate(trajs))
kernelcenterer = KernelCenterer()
kernelcenterer.fit(trajs)
y_ref1 = kernelcentererr.transform(trajs[0])
y1 = kernelcenterer.transform(trajs)[0]
np.testing.assert_array_almost_equal(y_ref1, y1)
def test_kernelcenterer_vs_sklearn():
# Compare msmbuilder.preprocessing.KernelCenterer
# with sklearn.preprocessing.KernelCenterer
kernelcentererr = KernelCentererR()
kernelcentererr.fit(np.concatenate(trajs))
kernelcenterer = KernelCenterer()
kernelcenterer.fit(trajs)
y_ref1 = kernelcentererr.transform(trajs[0])
y1 = kernelcenterer.transform(trajs)[0]
np.testing.assert_array_almost_equal(y_ref1, y1)
def chooseKernel(data, kerneltype='euclidean'):
if kerneltype == 'euclidean':
K = np.divide(1, (1 + pairwise_distances(data, metric="euclidean")))
elif kerneltype == 'cosine':
K = (pairwise.cosine_kernel(data))
elif kerneltype == 'laplacian':
K = (pairwise.laplacian_kernel(data))
elif kerneltype == 'linear':
K = (pairwise.linear_kernel(data))
elif kerneltype == 'polynomial_kernel':
K = (pairwise.polynomial_kernel(data))
elif kerneltype == 'jaccard':
K = 1 - distance.cdist(data, data, metric='jaccard')
scaler = KernelCenterer().fit(K)
return (scaler.transform(K))
def cv_mkl(kernel_list, labels, mkl, n_folds, dataset, data):
n_sample, n_labels = labels.shape
n_km = len(kernel_list)
tags = np.loadtxt("../data/cv/"+data+".cv")
for i in range(1,n_folds+1):
print "Test fold %d" %i
res_f = "../svm_result/weights/"+dataset+"_fold_%d_%s.weights" % (i,mkl)
para_f = "../svm_result/upperbound/"+dataset+"_fold_%d_%s.ubound" % (i,mkl)
test = np.array(tags == i)
train = np.array(~test)
train_y = labels[train,:]
test_y = labels[test,:]
n_train = len(train_y)
n_test = len(test_y)
train_km_list = []
# all train kernels are nomalized and centered
for km in kernel_list:
kc = KernelCenterer()
train_km = km[np.ix_(train, train)]
# center train and test kernels
kc.fit(train_km)
train_km_c = kc.transform(train_km)
train_km_list.append(train_km_c)
if mkl == 'UNIMKL':
res = UNIMKL(train_km_list, train_y)
np.savetxt(res_f, res)
if mkl == 'ALIGNF2':
res = alignf2(train_km_list, train_y, data)
np.savetxt(res_f, res)
if mkl.find('ALIGNF2SOFT') != -1:
bestC, res = ALIGNF2SOFT(train_km_list, train_y, i, tags, data)
np.savetxt(res_f, res)
np.savetxt(para_f, bestC)
if mkl == "TSMKL":
W = np.zeros((n_km, n_labels))
for j in xrange(n_labels):
print "..label",j
W[:,j] = TSMKL(train_km_list, train_y[:,j])
res_f = "../svm_result/weights/"+dataset+"_fold_%d_%s.weights" % (i,mkl)
np.savetxt(res_f, W)
class KernelPca:
# beta: ガウスカーネルパラメータ
def __init__(self, beta):
self.beta = beta
self.centerer = KernelCenterer()
# gauss kernel
def __kernel(self, x1, x2):
return np.exp(-self.beta * np.linalg.norm(x1 - x2)**2)
# データを入力して主成分ベクトルを計算する
# shape(X) = (N, M)
# n: 抽出する主成分の数
def fit_transform(self, X, n):
self.X = X
# グラム行列
N = X.shape[0]
K = np.array([[self.__kernel(X[i], X[j]) for j in range(N)]
for i in range(N)])
# 中心化
K = self.centerer.fit_transform(K)
# eighは固有値の昇順で出力される
vals, vecs = np.linalg.eigh(K)
vals = vals[::-1]
vecs = vecs[:, ::-1]
# 特異値と左特異ベクトル、上位n個
self.sigma = np.sqrt(vals[:n]) # (n)
self.a = np.array(vecs[:, :n]) # (N,n)
return self.sigma * self.a # (N,n)
# xの主成分表示を返す
# shape(x)=(Nx, M)
def transform(self, x):
# グラム行列
N = self.X.shape[0]
Nx = x.shape[0]
K = np.array([[self.__kernel(x[i], self.X[j]) for j in range(N)]
for i in range(Nx)]) # (Nx,N)
# 中心化
K = self.centerer.transform(K)
# 主成分を計算
return K.dot(self.a) / self.sigma # (Nx,n)
def test_center_kernel():
"""Test that KernelCenterer is equivalent to Scaler in feature space"""
X_fit = np.random.random((5, 4))
scaler = Scaler(with_std=False)
scaler.fit(X_fit)
X_fit_centered = scaler.transform(X_fit)
K_fit = np.dot(X_fit, X_fit.T)
# center fit time matrix
centerer = KernelCenterer()
K_fit_centered = np.dot(X_fit_centered, X_fit_centered.T)
K_fit_centered2 = centerer.fit_transform(K_fit)
assert_array_almost_equal(K_fit_centered, K_fit_centered2)
# center predict time matrix
X_pred = np.random.random((2, 4))
K_pred = np.dot(X_pred, X_fit.T)
X_pred_centered = scaler.transform(X_pred)
K_pred_centered = np.dot(X_pred_centered, X_fit_centered.T)
K_pred_centered2 = centerer.transform(K_pred)
assert_array_almost_equal(K_pred_centered, K_pred_centered2)
class kc():
def __init__(self, cols, metric):
self.columns = cols
self.metric = metric
self.model = KernelCenterer()
def fit(self, data):
k = pairwise_kernels(data[self.columns], metric=self.metric)
self.model.fit(k)
def fit_transform(self, data):
k = pairwise_kernels(data[self.columns], metric=self.metric)
transformed = self.model.fit_transform(k)
for idx in range(len(self.columns)):
data[self.columns[idx]] = transformed[:, idx]
return data
def transform(self, data):
k = pairwise_kernels(data[self.columns], metric=self.metric)
transformed = self.model.transform(k)
for idx in range(len(self.columns)):
data[self.columns[idx]] = transformed[:, idx]
return data
if __name__ == "__main__":
classes = generate_spike_classes(1, 2)
train = generate_spike_times(classes)
test = generate_spike_times(classes)
rasterPlot(train)
K = compute_K_matrix(train)
###############################
# N = K.shape[0]
# H = np.eye(N) - np.tile(1./N, [N, N]);
# Kc = np.dot(np.dot(H, K), H)
kcenterer = KernelCenterer() #
kcenterer.fit(K) # Center Kernel Matrix
Kc = kcenterer.transform(K) #
###############################
D, E = eig(Kc)
proj = np.dot(Kc, E[:, 0:2])
################################ Center test
Kt = compute_K_matrix(train, test)
# M = Kt.shape[0]
# A = np.tile(K.sum(axis=0), [M, 1]) / N
# B = np.tile(Kt.sum(axis=1),[N, 1]) /N
# Kc2 = Kt - A - B + K.sum()/ N**2;
Kc2 = kcenterer.transform(Kt)
proj2 = np.dot(Kc2, E[:, 0:2])
# kpca = KernelPCA(kernel="precomputed", n_components=2)
# kpca.fit(Kc)
if __name__ == '__main__':
classes = generate_spike_classes(1, 2)
train = generate_spike_times(classes)
test = generate_spike_times(classes)
rasterPlot(train)
K = compute_K_matrix(train)
###############################
#N = K.shape[0]
#H = np.eye(N) - np.tile(1./N, [N, N]);
#Kc = np.dot(np.dot(H, K), H)
kcenterer = KernelCenterer() #
kcenterer.fit(K) # Center Kernel Matrix
Kc = kcenterer.transform(K) #
###############################
D, E = eig(Kc)
proj = np.dot(Kc, E[:, 0:2])
################################ Center test
Kt = compute_K_matrix(train, test)
#M = Kt.shape[0]
#A = np.tile(K.sum(axis=0), [M, 1]) / N
#B = np.tile(Kt.sum(axis=1),[N, 1]) /N
#Kc2 = Kt - A - B + K.sum()/ N**2;
Kc2 = kcenterer.transform(Kt)
proj2 = np.dot(Kc2, E[:, 0:2])
#kpca = KernelPCA(kernel="precomputed", n_components=2)
#kpca.fit(Kc)
class KernelECA(BaseEstimator, TransformerMixin):
"""Kernel Entropy component analysis (KECA)
Non-linear dimensionality reduction through the use of kernels (see
:ref:`metrics`).
Parameters
----------
n_components: int or None
Number of components. If None, all non-zero components are kept.
kernel: "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed"
Kernel.
Default: "linear"
degree : int, default=3
Degree for poly kernels. Ignored by other kernels.
gamma : float, optional
Kernel coefficient for rbf and poly kernels. Default: 1/n_features.
Ignored by other kernels.
coef0 : float, optional
Independent term in poly and sigmoid kernels.
Ignored by other kernels.
kernel_params : mapping of string to any, optional
Parameters (keyword arguments) and values for kernel passed as
callable object. Ignored by other kernels.
eigen_solver: string ['auto'|'dense'|'arpack']
Select eigensolver to use. If n_components is much less than
the number of training samples, arpack may be more efficient
than the dense eigensolver.
tol: float
convergence tolerance for arpack.
Default: 0 (optimal value will be chosen by arpack)
max_iter : int
maximum number of iterations for arpack
Default: None (optimal value will be chosen by arpack)
random_state : int seed, RandomState instance, or None, default : None
A pseudo random number generator used for the initialization of the
residuals when eigen_solver == 'arpack'.
Attributes
----------
lambdas_ :
Eigenvalues of the centered kernel matrix
alphas_ :
Eigenvectors of the centered kernel matrix
dual_coef_ :
Inverse transform matrix
X_transformed_fit_ :
Projection of the fitted data on the kernel entropy components
References
----------
Kernel ECA based on:
(c) Robert Jenssen, University of Tromso, Norway, 2010
R. Jenssen, "Kernel Entropy Component Analysis,"
IEEE Trans. Patt. Anal. Mach. Intel., 32(5), 847-860, 2010.
"""
def __init__(self, n_components=None, kernel="linear",
gamma=None, degree=3, coef0=1, kernel_params=None, eigen_solver='auto',
tol=0, max_iter=None, random_state=None,center=False):
self.n_components = n_components
self._kernel = kernel
self.kernel_params = kernel_params
self.gamma = gamma
self.degree = degree
self.coef0 = coef0
self.eigen_solver = eigen_solver
self.tol = tol
self.max_iter = max_iter
self.random_state = random_state
self._centerer = KernelCenterer()
self.center = center
@property
def _pairwise(self):
return self.kernel == "precomputed"
def _get_kernel(self, X, Y=None):
if callable(self._kernel):
params = self.kernel_params or {}
else:
params = {"gamma": self.gamma,
"degree": self.degree,
"coef0": self.coef0}
return pairwise_kernels(X, Y, metric=self._kernel,
filter_params=True, **params)
def _fit_transform(self, K):
""" Fit's using kernel K"""
# center kernel
if self.center == True:
K = self._centerer.fit_transform(K)
X_transformed = self.kernelECA(K=K)
self.X_transformed = X_transformed
return K
def fit(self, X, y=None):
"""Fit the model from data in X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
self : object
Returns the instance itself.
"""
K = self._get_kernel(X)
self._fit_transform(K)
self.X_fit_ = X
return self
def fit_transform(self, X, y=None, **params):
"""Fit the model from data in X and transform X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
X_new: array-like, shape (n_samples, n_components)
"""
self.fit(X, **params)
X_transformed= self.X_transformed
return X_transformed
def transform(self, X):
"""Transform X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Returns
-------
X_new: array-like, shape (n_samples, n_components)
"""
check_is_fitted(self, 'X_fit_')
K = self._centerer.transform(self._get_kernel(X, self.X_fit_))
return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_))
def inverse_transform(self, X):
raise NotImplementedError("Function inverse_transform is not implemented.")
# here are the helper functions => to integrate in the code!
def kernelECA(self,K):
if self.n_components is None:
n_components = K.shape[0]
else:
n_components = min(K.shape[0], self.n_components)
# compute eigenvectors
self.lambdas_, self.alphas_ = linalg.eigh(K)
d = self.lambdas_
E = self.alphas_
# sort eigenvectors in descending order
D,E = self.sort_eigenvalues(d,E)
d = np.diag(D)
sorted_entropy_index,entropy = self.ECA(D,E)
Es = E[:,sorted_entropy_index]
ds = d[sorted_entropy_index]
Phi = np.zeros((K.shape[0],n_components))
for i in range(n_components):
Phi[:,i] = np.sqrt(ds[i]) * Es[:,i]
X_transformed = Phi
return X_transformed
def sort_eigenvalues(self,D,E):
d = D
indices = np.argsort(d)[::-1]
d = d[indices]
D = np.zeros((len(d),len(d)))
for i in range(len(d)):
D[i,i] = d[i]
E = E[:,indices]
return D,E
def ECA(self,D,E):
N = E.shape[0]
entropy = np.multiply(np.diag(D).T , (np.dot(np.ones((1,N)),E))**2)[0]
indices = np.argsort(entropy)[::-1]
entropy = entropy[indices]
return indices,entropy
class KernelECA(BaseEstimator, TransformerMixin):
"""Kernel Entropy component analysis (KECA)
Non-linear dimensionality reduction through the use of kernels (see
:ref:`metrics`).
Parameters
----------
n_components: int or None
Number of components. If None, all non-zero components are kept.
kernel: "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed"
Kernel.
Default: "linear"
degree : int, default=3
Degree for poly kernels. Ignored by other kernels.
gamma : float, optional
Kernel coefficient for rbf and poly kernels. Default: 1/n_features.
Ignored by other kernels.
coef0 : float, optional
Independent term in poly and sigmoid kernels.
Ignored by other kernels.
kernel_params : mapping of string to any, optional
Parameters (keyword arguments) and values for kernel passed as
callable object. Ignored by other kernels.
eigen_solver: string ['auto'|'dense'|'arpack']
Select eigensolver to use. If n_components is much less than
the number of training samples, arpack may be more efficient
than the dense eigensolver.
tol: float
convergence tolerance for arpack.
Default: 0 (optimal value will be chosen by arpack)
max_iter : int
maximum number of iterations for arpack
Default: None (optimal value will be chosen by arpack)
random_state : int seed, RandomState instance, or None, default : None
A pseudo random number generator used for the initialization of the
residuals when eigen_solver == 'arpack'.
Attributes
----------
lambdas_ :
Eigenvalues of the centered kernel matrix
alphas_ :
Eigenvectors of the centered kernel matrix
dual_coef_ :
Inverse transform matrix
X_transformed_fit_ :
Projection of the fitted data on the kernel entropy components
References
----------
Kernel ECA based on:
(c) Robert Jenssen, University of Tromso, Norway, 2010
R. Jenssen, "Kernel Entropy Component Analysis,"
IEEE Trans. Patt. Anal. Mach. Intel., 32(5), 847-860, 2010.
"""
def __init__(self,
n_components=None,
kernel="linear",
gamma=None,
degree=3,
coef0=1,
kernel_params=None,
eigen_solver='auto',
tol=0,
max_iter=None,
random_state=None,
center=False):
self.n_components = n_components
self._kernel = kernel
self.kernel_params = kernel_params
self.gamma = gamma
self.degree = degree
self.coef0 = coef0
self.eigen_solver = eigen_solver
self.tol = tol
self.max_iter = max_iter
self.random_state = random_state
self._centerer = KernelCenterer()
self.center = center
@property
def _pairwise(self):
return self.kernel == "precomputed"
def _get_kernel(self, X, Y=None):
if callable(self._kernel):
params = self.kernel_params or {}
else:
params = {
"gamma": self.gamma,
"degree": self.degree,
"coef0": self.coef0
}
return pairwise_kernels(X,
Y,
metric=self._kernel,
filter_params=True,
**params)
def _fit_transform(self, K):
""" Fit's using kernel K"""
# center kernel
if self.center == True:
K = self._centerer.fit_transform(K)
X_transformed = self.kernelECA(K=K)
self.X_transformed = X_transformed
return K
def fit(self, X, y=None):
"""Fit the model from data in X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
self : object
Returns the instance itself.
"""
K = self._get_kernel(X)
self._fit_transform(K)
self.X_fit_ = X
return self
def fit_transform(self, X, y=None, **params):
"""Fit the model from data in X and transform X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
X_new: array-like, shape (n_samples, n_components)
"""
self.fit(X, **params)
X_transformed = self.X_transformed
return X_transformed
def transform(self, X):
"""Transform X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Returns
-------
X_new: array-like, shape (n_samples, n_components)
"""
check_is_fitted(self, 'X_fit_')
K = self._centerer.transform(self._get_kernel(X, self.X_fit_))
return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_))
def inverse_transform(self, X):
raise NotImplementedError(
"Function inverse_transform is not implemented.")
# here are the helper functions => to integrate in the code!
def kernelECA(self, K):
if self.n_components is None:
n_components = K.shape[0]
else:
n_components = min(K.shape[0], self.n_components)
# compute eigenvectors
self.lambdas_, self.alphas_ = linalg.eigh(K)
d = self.lambdas_
E = self.alphas_
# sort eigenvectors in descending order
D, E = self.sort_eigenvalues(d, E)
d = np.diag(D)
sorted_entropy_index, entropy = self.ECA(D, E)
Es = E[:, sorted_entropy_index]
ds = d[sorted_entropy_index]
Phi = np.zeros((K.shape[0], n_components))
for i in range(n_components):
Phi[:, i] = np.sqrt(ds[i]) * Es[:, i]
X_transformed = Phi
return X_transformed
def sort_eigenvalues(self, D, E):
d = D
indices = np.argsort(d)[::-1]
d = d[indices]
D = np.zeros((len(d), len(d)))
for i in range(len(d)):
D[i, i] = d[i]
E = E[:, indices]
return D, E
def ECA(self, D, E):
N = E.shape[0]
entropy = np.multiply(np.diag(D).T, (np.dot(np.ones((1, N)), E))**2)[0]
indices = np.argsort(entropy)[::-1]
entropy = entropy[indices]
return indices, entropy
class MIDA(BaseEstimator, TransformerMixin):
"""Maximum independence domain adaptation
Args:
n_components (int): Number of components to keep.
kernel (str): "linear", "rbf", or "poly". Kernel to use for MIDA. Defaults to "linear".
mu (float): Hyperparameter of the l2 penalty. Defaults to 1.0.
eta (float): Hyperparameter of the label dependence. Defaults to 1.0.
augmentation (bool): Whether using covariates as augment features. Defaults to False.
kernel_params (dict or None): Parameters for the kernel. Defaults to None.
References:
[1] Yan, K., Kou, L. and Zhang, D., 2018. Learning domain-invariant subspace using domain features and
independence maximization. IEEE transactions on cybernetics, 48(1), pp.288-299.
"""
def __init__(
self,
n_components,
kernel="linear",
lambda_=1.0,
mu=1.0,
eta=1.0,
augmentation=False,
kernel_params=None,
):
self.n_components = n_components
self.kernel = kernel
self.mu = mu
self.eta = eta
self.augmentation = augmentation
if kernel_params is None:
self.kernel_params = {}
else:
self.kernel_params = kernel_params
self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1)
self._centerer = KernelCenterer()
self.x_fit = None
def _get_kernel(self, x, y=None):
if self.kernel in ["linear", "rbf", "poly"]:
params = self.kernel_params or {}
else:
raise ValueError("Pre-computed kernel not supported")
return pairwise_kernels(x,
y,
metric=self.kernel,
filter_params=True,
**params)
def fit(self, x, y=None, covariates=None):
"""
Args:
x : array-like. Input data, shape (n_samples, n_features)
y : array-like. Labels, shape (nl_samples,)
covariates : array-like. Domain co-variates, shape (n_samples, n_co-variates)
Note:
Unsupervised MIDA is performed if y is None.
Semi-supervised MIDA is performed is y is not None.
"""
if self.augmentation and type(covariates) == np.ndarray:
x = np.concatenate((x, covariates), axis=1)
# Kernel matrix
kernel_x = self._get_kernel(x)
kernel_x[np.isnan(kernel_x)] = 0
# Solve the optimization problem
self._fit(kernel_x, y, covariates)
self.x_fit = x
return self
def _fit(self, kernel_x, y, covariates=None):
"""solve MIDA
Args:
kernel_x: array-like, kernel matrix of input data x, shape (n_samples, n_samples)
y: array-like. Labels, shape (nl_samples,)
covariates: array-like. Domain co-variates, shape (n_samples, n_covariates)
Returns:
self
"""
n_samples = kernel_x.shape[0]
# Identity (unit) matrix
unit_mat = np.eye(n_samples)
# Centering matrix
ctr_mat = unit_mat - 1.0 / n_samples * np.ones((n_samples, n_samples))
kernel_x = self._centerer.fit_transform(kernel_x)
if type(covariates) == np.ndarray:
kernel_c = np.dot(covariates, covariates.T)
else:
kernel_c = np.zeros((n_samples, n_samples))
if y is not None:
n_labeled = y.shape[0]
if n_labeled > n_samples:
raise ValueError("Number of labels exceeds number of samples")
y_mat_ = self._label_binarizer.fit_transform(y)
y_mat = np.zeros((n_samples, y_mat_.shape[1]))
y_mat[:n_labeled, :] = y_mat_
ker_y = np.dot(y_mat, y_mat.T)
obj = multi_dot([
kernel_x,
self.mu * ctr_mat +
self.eta * multi_dot([ctr_mat, ker_y, ctr_mat]) -
multi_dot([ctr_mat, kernel_c, ctr_mat]),
kernel_x.T,
])
else:
obj = multi_dot([
kernel_x,
self.mu * ctr_mat - multi_dot([ctr_mat, kernel_c, ctr_mat]),
kernel_x.T
])
eig_values, eig_vectors = linalg.eigh(
obj,
subset_by_index=[n_samples - self.n_components, n_samples - 1])
idx_sorted = eig_values.argsort()[::-1]
self.eig_values_ = eig_values[idx_sorted]
self.U = eig_vectors[:, idx_sorted]
self.U = np.asarray(self.U, dtype=np.float)
return self
def fit_transform(self, x, y=None, covariates=None):
"""
Args:
x : array-like, shape (n_samples, n_features)
y : array-like, shape (n_samples,)
covariates : array-like, shape (n_samples, n_covariates)
Returns:
x_transformed : array-like, shape (n_samples, n_components)
"""
self.fit(x, y, covariates)
return self.transform(x, covariates)
def transform(self, x, covariates=None):
"""
Args:
x : array-like, shape (n_samples, n_features)
covariates : array-like, augmentation features, shape (n_samples, n_covariates)
Returns:
x_transformed : array-like, shape (n_samples, n_components)
"""
check_is_fitted(self, "x_fit")
if type(covariates) == np.ndarray and self.augmentation:
x = np.concatenate((x, covariates), axis=1)
kernel_x = self._centerer.transform(
pairwise_kernels(x,
self.x_fit,
metric=self.kernel,
filter_params=True,
**self.kernel_params))
return np.dot(kernel_x, self.U)
class KernelPCA(BaseEstimator, TransformerMixin):
"""Kernel Principal component analysis (KPCA)
Non-linear dimensionality reduction through the use of kernels (see
:ref:`metrics`).
Read more in the :ref:`User Guide <kernel_PCA>`.
Parameters
----------
n_components : int, default=None
Number of components. If None, all non-zero components are kept.
kernel : "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed"
Kernel. Default="linear".
gamma : float, default=1/n_features
Kernel coefficient for rbf, poly and sigmoid kernels. Ignored by other
kernels.
degree : int, default=3
Degree for poly kernels. Ignored by other kernels.
coef0 : float, default=1
Independent term in poly and sigmoid kernels.
Ignored by other kernels.
kernel_params : mapping of string to any, default=None
Parameters (keyword arguments) and values for kernel passed as
callable object. Ignored by other kernels.
alpha : int, default=1.0
Hyperparameter of the ridge regression that learns the
inverse transform (when fit_inverse_transform=True).
fit_inverse_transform : bool, default=False
Learn the inverse transform for non-precomputed kernels.
(i.e. learn to find the pre-image of a point)
eigen_solver : string ['auto'|'dense'|'arpack'], default='auto'
Select eigensolver to use. If n_components is much less than
the number of training samples, arpack may be more efficient
than the dense eigensolver.
tol : float, default=0
Convergence tolerance for arpack.
If 0, optimal value will be chosen by arpack.
max_iter : int, default=None
Maximum number of iterations for arpack.
If None, optimal value will be chosen by arpack.
remove_zero_eig : boolean, default=False
If True, then all components with zero eigenvalues are removed, so
that the number of components in the output may be < n_components
(and sometimes even zero due to numerical instability).
When n_components is None, this parameter is ignored and components
with zero eigenvalues are removed regardless.
random_state : int, RandomState instance or None, optional (default=None)
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used
by `np.random`. Used when ``eigen_solver`` == 'arpack'.
.. versionadded:: 0.18
copy_X : boolean, default=True
If True, input X is copied and stored by the model in the `X_fit_`
attribute. If no further changes will be done to X, setting
`copy_X=False` saves memory by storing a reference.
.. versionadded:: 0.18
n_jobs : int, default=1
The number of parallel jobs to run.
If `-1`, then the number of jobs is set to the number of CPU cores.
.. versionadded:: 0.18
Attributes
----------
lambdas_ : array, (n_components,)
Eigenvalues of the centered kernel matrix in decreasing order.
If `n_components` and `remove_zero_eig` are not set,
then all values are stored.
alphas_ : array, (n_samples, n_components)
Eigenvectors of the centered kernel matrix. If `n_components` and
`remove_zero_eig` are not set, then all components are stored.
dual_coef_ : array, (n_samples, n_features)
Inverse transform matrix. Set if `fit_inverse_transform` is True.
X_transformed_fit_ : array, (n_samples, n_components)
Projection of the fitted data on the kernel principal components.
X_fit_ : (n_samples, n_features)
The data used to fit the model. If `copy_X=False`, then `X_fit_` is
a reference. This attribute is used for the calls to transform.
References
----------
Kernel PCA was introduced in:
Bernhard Schoelkopf, Alexander J. Smola,
and Klaus-Robert Mueller. 1999. Kernel principal
component analysis. In Advances in kernel methods,
MIT Press, Cambridge, MA, USA 327-352.
"""
def __init__(self,
n_components=None,
kernel="linear",
gamma=None,
degree=3,
coef0=1,
kernel_params=None,
alpha=1.0,
fit_inverse_transform=False,
eigen_solver='auto',
tol=0,
max_iter=None,
remove_zero_eig=False,
random_state=None,
copy_X=True,
n_jobs=1):
if fit_inverse_transform and kernel == 'precomputed':
raise ValueError(
"Cannot fit_inverse_transform with a precomputed kernel.")
self.n_components = n_components
self.kernel = kernel
self.kernel_params = kernel_params
self.gamma = gamma
self.degree = degree
self.coef0 = coef0
self.alpha = alpha
self.fit_inverse_transform = fit_inverse_transform
self.eigen_solver = eigen_solver
self.remove_zero_eig = remove_zero_eig
self.tol = tol
self.max_iter = max_iter
self._centerer = KernelCenterer()
self.random_state = random_state
self.n_jobs = n_jobs
self.copy_X = copy_X
@property
def _pairwise(self):
return self.kernel == "precomputed"
def _get_kernel(self, X, Y=None):
if callable(self.kernel):
params = self.kernel_params or {}
else:
params = {
"gamma": self.gamma,
"degree": self.degree,
"coef0": self.coef0
}
return pairwise_kernels(X,
Y,
metric=self.kernel,
filter_params=True,
n_jobs=self.n_jobs,
**params)
def _fit_transform(self, K):
""" Fit's using kernel K"""
# center kernel
K = self._centerer.fit_transform(K)
if self.n_components is None:
n_components = K.shape[0]
else:
n_components = min(K.shape[0], self.n_components)
# compute eigenvectors
if self.eigen_solver == 'auto':
if K.shape[0] > 200 and n_components < 10:
eigen_solver = 'arpack'
else:
eigen_solver = 'dense'
else:
eigen_solver = self.eigen_solver
if eigen_solver == 'dense':
self.lambdas_, self.alphas_ = linalg.eigh(
K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1))
elif eigen_solver == 'arpack':
random_state = check_random_state(self.random_state)
# initialize with [-1,1] as in ARPACK
v0 = random_state.uniform(-1, 1, K.shape[0])
self.lambdas_, self.alphas_ = eigsh(K,
n_components,
which="LA",
tol=self.tol,
maxiter=self.max_iter,
v0=v0)
# sort eigenvectors in descending order
indices = self.lambdas_.argsort()[::-1]
self.lambdas_ = self.lambdas_[indices]
self.alphas_ = self.alphas_[:, indices]
# remove eigenvectors with a zero eigenvalue
if self.remove_zero_eig or self.n_components is None:
self.alphas_ = self.alphas_[:, self.lambdas_ > 0]
self.lambdas_ = self.lambdas_[self.lambdas_ > 0]
return K
def _fit_inverse_transform(self, X_transformed, X):
if hasattr(X, "tocsr"):
raise NotImplementedError("Inverse transform not implemented for "
"sparse matrices!")
n_samples = X_transformed.shape[0]
K = self._get_kernel(X_transformed)
K.flat[::n_samples + 1] += self.alpha
self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True)
self.X_transformed_fit_ = X_transformed
def fit(self, X, y=None):
"""Fit the model from data in X.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
self : object
Returns the instance itself.
"""
X = check_array(X, accept_sparse='csr', copy=self.copy_X)
K = self._get_kernel(X)
self._fit_transform(K)
if self.fit_inverse_transform:
sqrt_lambdas = np.diag(np.sqrt(self.lambdas_))
X_transformed = np.dot(self.alphas_, sqrt_lambdas)
self._fit_inverse_transform(X_transformed, X)
self.X_fit_ = X
return self
def fit_transform(self, X, y=None, **params):
"""Fit the model from data in X and transform X.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
X_new : array-like, shape (n_samples, n_components)
"""
self.fit(X, **params)
X_transformed = self.alphas_ * np.sqrt(self.lambdas_)
if self.fit_inverse_transform:
self._fit_inverse_transform(X_transformed, X)
return self.alphas_, self.lambdas_
def transform(self, X):
"""Transform X.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Returns
-------
X_new : array-like, shape (n_samples, n_components)
"""
check_is_fitted(self, 'X_fit_')
K = self._centerer.transform(self._get_kernel(X, self.X_fit_))
return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_))
def get_eigen(self):
return self.alphas_, self.lambdas_
def inverse_transform(self, X):
"""Transform X back to original space.
Parameters
----------
X : array-like, shape (n_samples, n_components)
Returns
-------
X_new : array-like, shape (n_samples, n_features)
References
----------
"Learning to Find Pre-Images", G BakIr et al, 2004.
"""
if not self.fit_inverse_transform:
raise NotFittedError("The fit_inverse_transform parameter was not"
" set to True when instantiating and hence "
"the inverse transform is not available.")
K = self._get_kernel(X, self.X_transformed_fit_)
return np.dot(K, self.dual_coef_)
class KernelPCA(TransformerMixin, BaseEstimator):
"""Kernel Principal component analysis (KPCA)
Non-linear dimensionality reduction through the use of kernels (see
:ref:`metrics`).
Read more in the :ref:`User Guide <kernel_PCA>`.
Parameters
----------
n_components : int, default=None
Number of components. If None, all non-zero components are kept.
kernel : "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed"
Kernel. Default="linear".
gamma : float, default=1/n_features
Kernel coefficient for rbf, poly and sigmoid kernels. Ignored by other
kernels.
degree : int, default=3
Degree for poly kernels. Ignored by other kernels.
coef0 : float, default=1
Independent term in poly and sigmoid kernels.
Ignored by other kernels.
kernel_params : mapping of string to any, default=None
Parameters (keyword arguments) and values for kernel passed as
callable object. Ignored by other kernels.
alpha : int, default=1.0
Hyperparameter of the ridge regression that learns the
inverse transform (when fit_inverse_transform=True).
fit_inverse_transform : bool, default=False
Learn the inverse transform for non-precomputed kernels.
(i.e. learn to find the pre-image of a point)
eigen_solver : string ['auto'|'dense'|'arpack'], default='auto'
Select eigensolver to use. If n_components is much less than
the number of training samples, arpack may be more efficient
than the dense eigensolver.
tol : float, default=0
Convergence tolerance for arpack.
If 0, optimal value will be chosen by arpack.
max_iter : int, default=None
Maximum number of iterations for arpack.
If None, optimal value will be chosen by arpack.
remove_zero_eig : boolean, default=False
If True, then all components with zero eigenvalues are removed, so
that the number of components in the output may be < n_components
(and sometimes even zero due to numerical instability).
When n_components is None, this parameter is ignored and components
with zero eigenvalues are removed regardless.
random_state : int, RandomState instance or None, optional (default=None)
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used
by `np.random`. Used when ``eigen_solver`` == 'arpack'.
sklearn. versionadded:: 0.18
copy_X : boolean, default=True
If True, input X is copied and stored by the model in the `X_fit_`
attribute. If no further changes will be done to X, setting
`copy_X=False` saves memory by storing a reference.
sklearn. versionadded:: 0.18
n_jobs : int or None, optional (default=None)
The number of parallel jobs to run.
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
for more details.
sklearn. versionadded:: 0.18
Attributes
----------
lambdas_ : array, (n_components,)
Eigenvalues of the centered kernel matrix in decreasing order.
If `n_components` and `remove_zero_eig` are not set,
then all values are stored.
alphas_ : array, (n_samples, n_components)
Eigenvectors of the centered kernel matrix. If `n_components` and
`remove_zero_eig` are not set, then all components are stored.
dual_coef_ : array, (n_samples, n_features)
Inverse transform matrix. Only available when
``fit_inverse_transform`` is True.
X_transformed_fit_ : array, (n_samples, n_components)
Projection of the fitted data on the kernel principal components.
Only available when ``fit_inverse_transform`` is True.
X_fit_ : (n_samples, n_features)
The data used to fit the model. If `copy_X=False`, then `X_fit_` is
a reference. This attribute is used for the calls to transform.
Examples
--------
>>> from sklearn.datasets import load_digits
>>> from sklearn.decomposition import KernelPCA
>>> X, _ = load_digits(return_X_y=True)
>>> transformer = KernelPCA(n_components=7, kernel='linear')
>>> X_transformed = transformer.fit_transform(X)
>>> X_transformed.shape
(1797, 7)
References
----------
Kernel PCA was introduced in:
Bernhard Schoelkopf, Alexander J. Smola,
and Klaus-Robert Mueller. 1999. Kernel principal
component analysis. In Advances in kernel methods,
MIT Press, Cambridge, MA, USA 327-352.
"""
def __init__(self,
n_components=None,
kernel="linear",
gamma=None,
degree=3,
coef0=1,
kernel_params=None,
alpha=1.0,
fit_inverse_transform=False,
eigen_solver='auto',
tol=0,
max_iter=None,
remove_zero_eig=False,
random_state=None,
copy_X=True,
n_jobs=None):
if fit_inverse_transform and kernel == 'precomputed':
raise ValueError(
"Cannot fit_inverse_transform with a precomputed kernel.")
self.n_components = n_components
self.kernel = kernel
self.kernel_params = kernel_params
self.gamma = gamma
self.degree = degree
self.coef0 = coef0
self.alpha = alpha
self.fit_inverse_transform = fit_inverse_transform
self.eigen_solver = eigen_solver
self.remove_zero_eig = remove_zero_eig
self.tol = tol
self.max_iter = max_iter
self.random_state = random_state
self.n_jobs = n_jobs
self.copy_X = copy_X
@property
def _pairwise(self):
return self.kernel == "precomputed"
def _get_kernel(self, X, Y=None):
if callable(self.kernel):
params = self.kernel_params or {}
else:
params = {
"gamma": self.gamma,
"degree": self.degree,
"coef0": self.coef0
}
return pairwise_kernels(X,
Y,
metric=self.kernel,
filter_params=True,
n_jobs=self.n_jobs,
**params)
def _fit_transform(self, K):
""" Fit's using kernel K"""
# center kernel
K = self._centerer.fit_transform(K)
if self.n_components is None:
n_components = K.shape[0]
else:
n_components = min(K.shape[0], self.n_components)
# compute eigenvectors
if self.eigen_solver == 'auto':
if K.shape[0] > 200 and n_components < 10:
eigen_solver = 'arpack'
else:
eigen_solver = 'dense'
else:
eigen_solver = self.eigen_solver
if eigen_solver == 'dense':
self.lambdas_, self.alphas_ = linalg.eigh(
K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1))
elif eigen_solver == 'arpack':
random_state = check_random_state(self.random_state)
# initialize with [-1,1] as in ARPACK
v0 = random_state.uniform(-1, 1, K.shape[0])
self.lambdas_, self.alphas_ = eigsh(K,
n_components,
which="LA",
tol=self.tol,
maxiter=self.max_iter,
v0=v0)
# make sure that the eigenvalues are ok and fix numerical issues
self.lambdas_ = _check_psd_eigenvalues(self.lambdas_,
enable_warnings=False)
# flip eigenvectors' sign to enforce deterministic output
self.alphas_, _ = svd_flip(self.alphas_, np.empty_like(self.alphas_).T)
# sort eigenvectors in descending order
indices = self.lambdas_.argsort()[::-1]
self.lambdas_ = self.lambdas_[indices]
self.alphas_ = self.alphas_[:, indices]
# remove eigenvectors with a zero eigenvalue (null space) if required
if self.remove_zero_eig or self.n_components is None:
self.alphas_ = self.alphas_[:, self.lambdas_ > 0]
self.lambdas_ = self.lambdas_[self.lambdas_ > 0]
# Maintenance note on Eigenvectors normalization
# ----------------------------------------------
# there is a link between
# the eigenvectors of K=Phi(X)'Phi(X) and the ones of Phi(X)Phi(X)'
# if v is an eigenvector of K
# then Phi(X)v is an eigenvector of Phi(X)Phi(X)'
# if u is an eigenvector of Phi(X)Phi(X)'
# then Phi(X)'u is an eigenvector of Phi(X)Phi(X)'
#
# At this stage our self.alphas_ (the v) have norm 1, we need to scale
# them so that eigenvectors in kernel feature space (the u) have norm=1
# instead
#
# We COULD scale them here:
# self.alphas_ = self.alphas_ / np.sqrt(self.lambdas_)
#
# But choose to perform that LATER when needed, in `fit()` and in
# `transform()`.
return K
def _fit_inverse_transform(self, X_transformed, X):
if hasattr(X, "tocsr"):
raise NotImplementedError("Inverse transform not implemented for "
"sparse matrices!")
n_samples = X_transformed.shape[0]
K = self._get_kernel(X_transformed)
K.flat[::n_samples + 1] += self.alpha
self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True)
self.X_transformed_fit_ = X_transformed
def fit(self, X, y=None):
"""Fit the model from data in X.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
self : object
Returns the instance itself.
"""
X = check_array(X, accept_sparse='csr', copy=self.copy_X)
self._centerer = KernelCenterer()
K = self._get_kernel(X)
self.K = K
self._fit_transform(K)
if self.fit_inverse_transform:
# no need to use the kernel to transform X, use shortcut expression
X_transformed = self.alphas_ * np.sqrt(self.lambdas_)
self._fit_inverse_transform(X_transformed, X)
self.X_fit_ = X
return self
def fit_transform(self, X, y=None, **params):
"""Fit the model from data in X and transform X.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
X_new : array-like, shape (n_samples, n_components)
"""
self.fit(X, **params)
# no need to use the kernel to transform X, use shortcut expression
X_transformed = self.alphas_ * np.sqrt(self.lambdas_)
if self.fit_inverse_transform:
self._fit_inverse_transform(X_transformed, X)
return X_transformed
def transform(self, X):
"""Transform X.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Returns
-------
X_new : array-like, shape (n_samples, n_components)
"""
check_is_fitted(self)
# Compute centered gram matrix between X and training data X_fit_
K = self._centerer.transform(self._get_kernel(X, self.X_fit_))
# scale eigenvectors (properly account for null-space for dot product)
non_zeros = np.flatnonzero(self.lambdas_)
scaled_alphas = np.zeros_like(self.alphas_)
scaled_alphas[:, non_zeros] = (self.alphas_[:, non_zeros] /
np.sqrt(self.lambdas_[non_zeros]))
# Project with a scalar product between K and the scaled eigenvectors
return np.dot(K, scaled_alphas)
def inverse_transform(self, X):
"""Transform X back to original space.
Parameters
----------
X : array-like, shape (n_samples, n_components)
Returns
-------
X_new : array-like, shape (n_samples, n_features)
References
----------
"Learning to Find Pre-Images", G BakIr et al, 2004.
"""
if not self.fit_inverse_transform:
raise NotFittedError("The fit_inverse_transform parameter was not"
" set to True when instantiating and hence "
"the inverse transform is not available.")
K = self._get_kernel(X, self.X_transformed_fit_)
return np.dot(K, self.dual_coef_)
def ovkr_mkl(kernel_list, labels, mkl, n_folds, dataset, data):
n_sample, n_labels = labels.shape
n_km = len(kernel_list)
tags = np.loadtxt("../data/cv/"+data+".cv")
#tags = np.array(range(n_sample)) % n_folds + 1
#np.random.seed(1234)
#np.random.shuffle(tags)
pred = np.zeros((n_sample, n_labels))
# Run for each fold
for i in range(1,n_folds+1):
print "Test fold %d" %i
res_f = "../ovkr_result/weights/"+dataset+"_fold_%d_%s.weights" % (i,mkl)
# divide data
test = np.array(tags == i)
train = np.array(~test)
train_y = labels[train,:]
test_y = labels[test,:]
n_train = len(train_y)
n_test = len(test_y)
train_km_list = []
test_km_list = []
for km in kernel_list:
kc = KernelCenterer()
train_km = km[np.ix_(train, train)]
test_km = km[np.ix_(test, train)]
# center train and test kernels
kc.fit(train_km)
train_km_c = kc.transform(train_km)
test_km_c = kc.transform(test_km)
train_km_list.append(train_km_c)
test_km_list.append(test_km_c)
if mkl == 'UNIMKL':
wei = UNIMKL(n_km, n_labels)
else:
wei = np.loadtxt(res_f, ndmin=2)
normw = np.linalg.norm(wei)
uni = np.ones(n_km) / np.linalg.norm(np.ones(n_km))
if normw == 0:
wei[:,0] = uni
else:
wei[:,0] = wei[:,0] / normw
train_ckm = np.zeros((n_train,n_train))
for t in range(n_km):
train_ckm += wei[t,0]*train_km_list[t]
# combine train and test kernel using learned weights
test_ckm = np.zeros(test_km_list[0].shape)
for t in range(n_km):
test_ckm = test_ckm + wei[t,0]*test_km_list[t]
AP = OVKR_train_CV(train_ckm, train_y, tags[train])
pred_label = OVKR_test(test_ckm, AP)
pred[test, :] = pred_label
return pred
def svm_mkl(kernel_list, labels, mkl, n_folds, dataset, data):
n_sample, n_labels = labels.shape
n_km = len(kernel_list)
tags = np.loadtxt("../data/cv/"+data+".cv")
pred = np.zeros((n_sample, n_labels))
# Run for each fold
for i in range(1,n_folds+1):
print "Test fold %d" %i
res_f = "../svm_result/weights/"+dataset+"_fold_%d_%s.weights" % (i,mkl)
# divide data
test = np.array(tags == (i+1 if i+1<6 else 1))
train = np.array(~test)
train_y = labels[train,:]
test_y = labels[test,:]
n_train = len(train_y)
n_test = len(test_y)
train_km_list = []
test_km_list = []
for km in kernel_list:
kc = KernelCenterer()
train_km = km[np.ix_(train, train)]
test_km = km[np.ix_(test, train)]
# center train and test kernels
kc.fit(train_km)
train_km_c = kc.transform(train_km)
test_km_c = kc.transform(test_km)
train_km_list.append(train_km_c)
test_km_list.append(test_km_c)
if mkl == 'UNIMKL':
wei = UNIMKL(n_km, n_labels)
else:
wei = np.loadtxt(res_f, ndmin=2)
# Normalized weights
normw = np.linalg.norm(wei, 2, 0)
uni = np.ones(n_km) / np.linalg.norm(np.ones(n_km))
for t in xrange(n_labels):
if normw[t] == 0: # collapsed solution
wei[:,t] = uni
else:
wei[:,t] = wei[:,t] / normw[t]
for j in range(n_labels):
tr_y = train_y[:,j]
te_y = test_y[:,j]
if wei.shape[1] == 1:
wj = wei[:,0]
else:
wj = wei[:,j]
ckm = np.zeros((n_train,n_train))
for t in range(n_km):
ckm = ckm + wj[t]*train_km_list[t]
# combine train and test kernel using learned weights
train_ckm = ckm
test_ckm = np.zeros(test_km_list[0].shape)
for t in range(n_km):
test_ckm = test_ckm + wj[t]*test_km_list[t]
pred_label = svm(train_ckm, test_ckm, tr_y, te_y, tags[train], i)
pred[test, j] = pred_label
return pred
def cls(mkl):
for data in datasets:
print "####################"
print '# ',data
print "####################"
# consider labels with more than 2%
t = 0.02
datadir = '../data/'
km_dir = datadir + data + "/"
if data == 'Fingerprint':
kernels = ['PPKr', 'NB','CP2','NI','LB','CPC','RLB','LC','LI','CPK','RLI','CSC']
km_list = []
y = np.loadtxt(km_dir+"y.txt",ndmin=2)
p = np.sum(y==1,0)/float(y.shape[0])
y = y[:,p>t]
for k in kernels:
km_f = datadir + data + ("/%s.txt" % k)
km_list.append(normalize_km(np.loadtxt(km_f)))
pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl)
pred = svm_mkl(km_list, y, mkl, 5, data,data)
np.savetxt(pred_f, pred, fmt="%d")
elif data == 'plant' or data == 'psortPos' or data == 'psortNeg':
y = loadmat(km_dir+"label_%s.mat" % data)['y'].ravel()
km_list = []
fs = commands.getoutput('ls %skern\;substr*.mat' % km_dir).split("\n")
for f in fs:
km = loadmat(f)
km_list.append(km['K'])
fs = commands.getoutput('ls %skern\;phylpro*.mat' % km_dir).split("\n")
for f in fs:
km = loadmat(f)
km_list.append(km['K'])
fs = commands.getoutput('ls %skm_evalue*.mat' % km_dir).split("\n")
for f in fs:
km = loadmat(f)
km_list.append(km['K'])
n_samples = y.shape[0]
n_km = len(km_list)
y_pred = np.zeros(n_samples)
n_labels = 1
tags = np.loadtxt("../data/cv/"+data+".cv")
for fold in range(1,6):
test_ind = np.where(tags == fold)[0]
train_ind = np.where(tags != fold)[0]
train_km_list = []
test_km_list = []
train_y = y[train_ind]
test_y = y[test_ind]
n_train = len(train_ind)
n_test = len(test_ind)
w_f = "../svm_result/weights/"+data+"_fold_%d_%s.weights" % (fold,mkl)
if mkl == 'UNIMKL':
w = UNIMKL(n_km, n_labels).ravel()
else:
w = np.loadtxt(w_f, ndmin=2).ravel()
normw = np.linalg.norm(w, 2, 0)
uni = np.ones(n_km) / np.linalg.norm(np.ones(n_km))
if normw == 0:
w = uni
else:
w = w / normw
for km in km_list:
kc = KernelCenterer()
train_km = km[np.ix_(train_ind, train_ind)]
test_km = km[np.ix_(test_ind, train_ind)]
# center train and test kernels
kc.fit(train_km)
train_km_c = kc.transform(train_km)
test_km_c = kc.transform(test_km)
train_km_list.append(train_km_c)
test_km_list.append(test_km_c)
train_ckm = np.zeros((n_train,n_train))
for t in range(n_km):
train_ckm = train_ckm + w[t]*train_km_list[t]
test_ckm = np.zeros(test_km_list[0].shape)
for t in range(n_km):
test_ckm = test_ckm + w[t]*test_km_list[t]
C_range = [0.01,0.1,1,10,100]
param_grid = dict(C=C_range)
cv = StratifiedShuffleSplit(train_y,n_iter=5,test_size=0.2,random_state=42)
grid = GridSearchCV(SVC(kernel='precomputed'), param_grid=param_grid, cv=cv)
grid.fit(train_ckm, train_y)
bestC = grid.best_params_['C']
svm = SVC(kernel='precomputed', C=bestC)
svm.fit(train_ckm, train_y)
y_pred[test_ind] = svm.predict(test_ckm)
pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl)
np.savetxt(pred_f, y_pred, fmt="%d")
elif data in image_datasets:
y = np.loadtxt(km_dir+"y.txt",ndmin=2)
p = np.sum(y==1,0)/float(y.shape[0])
y = y[:,p>t]
linear_km_list = []
for i in range(1,16):
name = 'kernel_linear_%d.txt' % i
km_f = km_dir+name
km = np.loadtxt(km_f)
# normalize input kernel !!!!!!!!
linear_km_list.append(normalize_km(km))
pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl)
pred = svm_mkl(linear_km_list, y, mkl, 5, data,data)
np.savetxt(pred_f, pred, fmt="%d")
elif data == 'SPAMBASE':
y = np.loadtxt(km_dir+"y.txt",ndmin=2)
rbf_km_list = []
gammas = [2**-9, 2**-8, 2**-7, 2**-6, 2**-5, 2**-4, 2**-3]
X = np.loadtxt(km_dir+"x.txt")
scaler = preprocessing.StandardScaler().fit(X)
X = scaler.transform(X)
X = preprocessing.normalize(X)
for gamma in gammas:
km = rbf_kernel(X, gamma=gamma)
rbf_km_list.append(km)
pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl)
pred = svm_mkl(rbf_km_list, y, mkl, 5, data,data)
np.savetxt(pred_f, pred, fmt="%d")
else:
rbf_km_list = []
gammas = [2**-13,2**-11,2**-9,2**-7,2**-5,2**-3,2**-1,2**1,2**3]
X = np.loadtxt(km_dir+"x.txt")
scaler = preprocessing.StandardScaler().fit(X)
X = scaler.transform(X)
X = preprocessing.normalize(X)
y = np.loadtxt(km_dir+"y.txt")
p = np.sum(y==1,0)/float(y.shape[0])
y = y[:,p>t]
for gamma in gammas:
km = rbf_kernel(X, gamma=gamma)
# normalize input kernel !!!!!!!!
rbf_km_list.append(km)
pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl)
pred = svm_mkl(rbf_km_list, y, mkl, 5, data,data)
np.savetxt(pred_f, pred, fmt="%d")
class KernelPCA(BaseEstimator, TransformerMixin):
"""Kernel Principal component analysis (KPCA)
Non-linear dimensionality reduction through the use of kernels (see
:ref:`metrics`).
Parameters
----------
n_components: int or None
Number of components. If None, all non-zero components are kept.
kernel: "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed"
Kernel.
Default: "linear"
degree : int, default=3
Degree for poly kernels. Ignored by other kernels.
gamma : float, optional
Kernel coefficient for rbf and poly kernels. Default: 1/n_features.
Ignored by other kernels.
coef0 : float, optional
Independent term in poly and sigmoid kernels.
Ignored by other kernels.
kernel_params : mapping of string to any, optional
Parameters (keyword arguments) and values for kernel passed as
callable object. Ignored by other kernels.
alpha: int
Hyperparameter of the ridge regression that learns the
inverse transform (when fit_inverse_transform=True).
Default: 1.0
fit_inverse_transform: bool
Learn the inverse transform for non-precomputed kernels.
(i.e. learn to find the pre-image of a point)
Default: False
eigen_solver: string ['auto'|'dense'|'arpack']
Select eigensolver to use. If n_components is much less than
the number of training samples, arpack may be more efficient
than the dense eigensolver.
tol: float
convergence tolerance for arpack.
Default: 0 (optimal value will be chosen by arpack)
max_iter : int
maximum number of iterations for arpack
Default: None (optimal value will be chosen by arpack)
remove_zero_eig : boolean, default=True
If True, then all components with zero eigenvalues are removed, so
that the number of components in the output may be < n_components
(and sometimes even zero due to numerical instability).
When n_components is None, this parameter is ignored and components
with zero eigenvalues are removed regardless.
Attributes
----------
lambdas_ :
Eigenvalues of the centered kernel matrix
alphas_ :
Eigenvectors of the centered kernel matrix
evals_ : array[float], shape=(n_features)
All eigenvalues of centered kernel matrix
evecs_ : array[float, float], shape=(n_features, n_samples)
All eigenvectors of centered kernel matrix
dual_coef_ :
Inverse transform matrix
X_transformed_fit_ :
Projection of the fitted data on the kernel principal components
References
----------
Kernel PCA was introduced in:
Bernhard Schoelkopf, Alexander J. Smola,
and Klaus-Robert Mueller. 1999. Kernel principal
component analysis. In Advances in kernel methods,
MIT Press, Cambridge, MA, USA 327-352.
"""
def __init__(self, n_components=None, kernel="linear",
gamma=None, degree=3, coef0=1, kernel_params=None,
alpha=1.0, fit_inverse_transform=False, eigen_solver='auto',
tol=0, max_iter=None, remove_zero_eig=False):
if fit_inverse_transform and kernel == 'precomputed':
raise ValueError(
"Cannot fit_inverse_transform with a precomputed kernel.")
self.n_components = n_components
self.kernel = kernel
self.kernel_params = kernel_params
self.gamma = gamma
self.degree = degree
self.coef0 = coef0
self.alpha = alpha
self.fit_inverse_transform = fit_inverse_transform
self.eigen_solver = eigen_solver
self.remove_zero_eig = remove_zero_eig
self.tol = tol
self.max_iter = max_iter
self._centerer = KernelCenterer()
@property
def _pairwise(self):
return self.kernel == "precomputed"
def _get_kernel(self, X, Y=None):
if callable(self.kernel):
params = self.kernel_params or {}
else:
params = {"gamma": self.gamma,
"degree": self.degree,
"coef0": self.coef0}
return pairwise_kernels(X, Y, metric=self.kernel,
filter_params=True, **params)
def _fit_transform(self, K):
""" Fit's using kernel K"""
# center kernel
K = self._centerer.fit_transform(K)
if self.n_components is None:
n_components = K.shape[0]
else:
n_components = min(K.shape[0], self.n_components)
# compute eigenvectors
if self.eigen_solver == 'auto':
if K.shape[0] > 200 and n_components < 10:
eigen_solver = 'arpack'
else:
eigen_solver = 'dense'
else:
eigen_solver = self.eigen_solver
if eigen_solver == 'dense':
self.lambdas_, self.alphas_ = linalg.eigh(
K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1))
self.evals_, self.evecs_ = linalg.eigh(K)
elif eigen_solver == 'arpack':
self.lambdas_, self.alphas_ = eigsh(K, n_components,
which="LA",
tol=self.tol,
maxiter=self.max_iter)
# sort eigenvectors in descending order
indices = self.lambdas_.argsort()[::-1]
self.lambdas_ = self.lambdas_[indices]
self.alphas_ = self.alphas_[:, indices]
# remove eigenvectors with a zero eigenvalue
if self.remove_zero_eig or self.n_components is None:
self.alphas_ = self.alphas_[:, self.lambdas_ > 0]
self.lambdas_ = self.lambdas_[self.lambdas_ > 0]
return K
def _fit_inverse_transform(self, X_transformed, X):
if hasattr(X, "tocsr"):
raise NotImplementedError("Inverse transform not implemented for "
"sparse matrices!")
n_samples = X_transformed.shape[0]
K = self._get_kernel(X_transformed)
K.flat[::n_samples + 1] += self.alpha
self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True)
self.X_transformed_fit_ = X_transformed
def fit(self, X, y=None):
"""Fit the model from data in X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
self : object
Returns the instance itself.
"""
K = self._get_kernel(X)
self._fit_transform(K)
if self.fit_inverse_transform:
sqrt_lambdas = np.diag(np.sqrt(self.lambdas_))
X_transformed = np.dot(self.alphas_, sqrt_lambdas)
self._fit_inverse_transform(X_transformed, X)
self.X_fit_ = X
return self
def fit_transform(self, X, y=None, **params):
"""Fit the model from data in X and transform X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
X_new: array-like, shape (n_samples, n_components)
"""
self.fit(X, **params)
X_transformed = self.alphas_ * np.sqrt(self.lambdas_)
if self.fit_inverse_transform:
self._fit_inverse_transform(X_transformed, X)
return X_transformed
def transform(self, X):
"""Transform X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Returns
-------
X_new: array-like, shape (n_samples, n_components)
"""
check_is_fitted(self, 'X_fit_')
K = self._centerer.transform(self._get_kernel(X, self.X_fit_))
return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_))
def inverse_transform(self, X):
"""Transform X back to original space.
Parameters
----------
X: array-like, shape (n_samples, n_components)
Returns
-------
X_new: array-like, shape (n_samples, n_features)
References
----------
"Learning to Find Pre-Images", G BakIr et al, 2004.
"""
if not self.fit_inverse_transform:
raise NotFittedError("The fit_inverse_transform parameter was not"
" set to True when instantiating and hence "
"the inverse transform is not available.")
K = self._get_kernel(X, self.X_transformed_fit_)
return np.dot(K, self.dual_coef_)
#!/usr/bin/env python import numpy as np from sklearn.preprocessing import KernelCenterer from sklearn.metrics.pairwise import pairwise_kernels X = np.array([[ 1., -2., 2.], [ -2., 1., 3.], [ 4., 1., -2.]]) K = pairwise_kernels(X, metric='linear') transformer = KernelCenterer().fit(K) centered_K = transformer.transform(K) print(centered_K) H = np.eye(3) - (1.0/3)*np.ones((3,3)) centered_K = H.dot(K).dot(H) print(centered_K)
def ovkr_mkl(kernel_list, labels, mkl, n_folds, dataset, data):
n_sample, n_labels = labels.shape
n_km = len(kernel_list)
tags = np.loadtxt("../data/cv/"+data+".cv")
# Add noise to the output
noise_level = [0.005, 0.010, 0.015, 0.020, 0.025]
for nid in xrange(len(noise_level)):
noi = noise_level[nid]
print "noise", noi, nid
Y = addNoise(labels, noi)
pred = np.zeros((n_sample, n_labels))
pred_bin = np.zeros((n_sample, n_labels))
# Run for each fold
for i in range(1,n_folds+1):
print "Test fold %d" %i
res_f = "../ovkr_result/noisy_weights/"+dataset+"_fold_%d_%s_noise_%d.weights" % (i,mkl, nid)
# divide data
test = np.array(tags == i)
train = np.array(~test)
train_y = Y[train,:]
test_y = Y[test,:]
n_train = len(train_y)
n_test = len(test_y)
train_km_list = []
test_km_list = []
for km in kernel_list:
kc = KernelCenterer()
train_km = km[np.ix_(train, train)]
test_km = km[np.ix_(test, train)]
# center train and test kernels
kc.fit(train_km)
train_km_c = kc.transform(train_km)
test_km_c = kc.transform(test_km)
train_km_list.append(train_km_c)
test_km_list.append(test_km_c)
if mkl == 'UNIMKL':
wei = UNIMKL(n_km, n_labels)
else:
wei = np.loadtxt(res_f, ndmin=2)
normw = np.linalg.norm(wei)
uni = np.ones(n_km) / np.linalg.norm(np.ones(n_km))
if normw == 0:
wei[:,0] = uni
else:
wei[:,0] = wei[:,0] / normw
train_ckm = np.zeros((n_train,n_train))
for t in range(n_km):
train_ckm += wei[t,0]*train_km_list[t]
# combine train and test kernel using learned weights
test_ckm = np.zeros(test_km_list[0].shape)
for t in range(n_km):
test_ckm = test_ckm + wei[t,0]*test_km_list[t]
AP = OVKR_train_CV(train_ckm, train_y, tags[train])
pred_label = OVKR_test(test_ckm, AP)
pred[test, :] = pred_label
pred_real_f = "../ovkr_result/noisy_pred/%s_cvpred_%s_real_noise_%d.npy" % (data, mkl, nid)
np.save(pred_real_f, pred)
# Kernel PLS
if (FE_kPLS == 1):
d = pair.pairwise_distances(Xtrain,Xtrain)
aux = np.triu(d)
sigma = np.sqrt(np.mean(np.power(aux[aux!=0],2)*0.5))
gamma = 1/(2*sigma**2)
ktrain = pair.rbf_kernel(Xtrain,Xtrain,gamma)
ktest = pair.rbf_kernel(Xtest,Xtrain,gamma)
kcent = KernelCenterer()
kcent.fit(ktrain)
ktrain = kcent.transform(ktrain)
ktest = kcent.transform(ktest)
kpls = PLSRegression(n_components = n_comp)
kpls.fit(ktrain,Ytrain_m)
Xtrain = kpls.transform(ktrain)
Xtest = kpls.transform(ktest)
# Linear CCA Cannonical Correlation Análisis
if (FE_CCA == 1):
from sklearn.cross_decomposition import CCA
cca = CCA(n_components = n_class)
cca.fit(Xtrain,Ytrain_m)
Xtrain = cca.transform(Xtrain)
def ALIGNFSOFT(kernel_list, ky, y, test_fold, tags):
# Find best upper bound in CV and train on whole data
# Reutrn the weights
y = y.ravel()
n_km = len(kernel_list)
tag = np.array(tags)
tag = tag[tag!=test_fold]
remain_fold = np.unique(tag).tolist()
all_best_c = []
for validate_fold in remain_fold:
train = tag != validate_fold
validate = tag == validate_fold
# train on train fold ,validate on validate_fold.
# Do not use test fold. test fold used in outter cv
ky_train = ky[np.ix_(train, train)]
y_train = y[train]
y_validate = y[validate]
train_km_list = []
validate_km_list = []
n_train = len(y_train)
n_validate = len(y_validate)
for km in kernel_list:
kc = KernelCenterer()
train_km = km[np.ix_(train, train)]
validate_km = km[np.ix_(validate, train)]
# center train and validate kernels
train_km_c = kc.fit_transform(train_km)
train_km_list.append(train_km_c)
validate_km_c = kc.transform(validate_km)
validate_km_list.append(validate_km_c)
# if the label is too biased, SVM CV will fail, just return ALIGNF solution
if np.sum(y_train==1) > n_train-3 or np.sum(y_train==-1) > n_train-3:
return 1e8, ALIGNFSLACK(train_km_list, ky_train, 1e8)
Cs = np.exp2(np.array(range(-9,7))).tolist() + [1e8]
W = np.zeros((n_km, len(Cs)))
for i in xrange(len(Cs)):
W[:,i] = ALIGNFSLACK(train_km_list, ky_train, Cs[i])
W = W / np.linalg.norm(W, 2, 0)
f1 = np.zeros(len(Cs))
for i in xrange(len(Cs)):
train_ckm = np.zeros((n_train,n_train))
validate_ckm = np.zeros((n_validate,n_train))
w = W[:,i]
for j in xrange(n_km):
train_ckm += w[j]*train_km_list[j]
validate_ckm += w[j]*validate_km_list[j]
f1[i] = svm(train_ckm, validate_ckm, y_train, y_validate)
# return the first maximum
maxind = np.argmax(f1)
bestC = Cs[maxind]
all_best_c.append(bestC)
print f1
print "..Best C is", bestC
bestC = np.mean(all_best_c)
print "..Take the average best upper bound", bestC
# use the best upper bound to solve ALIGNFSOFT
return bestC, ALIGNFSLACK(kernel_list, ky, bestC)
plt.xlim(1,np.amax(nComponents))
plt.title('kPCA accuracy')
plt.xlabel('Number of components')
plt.ylabel('accuracy')
plt.xlim([500,1500])
plt.legend (['LDA'],loc='lower right')
plt.grid(True)
if(0):
# K-PCA second round
ktrain = pair.rbf_kernel(Xtrain,Xtrain,gamma)
ktest = pair.rbf_kernel(Xtest,Xtrain,gamma)
kcent = KernelCenterer()
kcent.fit(ktrain)
ktrain = kcent.transform(ktrain)
ktest = kcent.transform(ktest)
kpca = PCA()
kpca.fit_transform(ktrain)
cumvarkPCA2 = np.cumsum(kpca.explained_variance_ratio_[0:220])
# Calculate classifiation scores for each component
nComponents = np.arange(1,nFeatures)
kpcaScores2 = np.zeros((5,np.alen(nComponents)))
for i,n in enumerate(nComponents):
kpca2 = PCA(n_components=n)
kpca2.fit(ktrain)
XtrainT = kpca2.transform(ktrain)
XtestT = kpca2.transform(ktest)
kpcaScores2[:,i] = util.classify(XtrainT,XtestT,labelsTrain,labelsTest)
class KernelPCA(BaseEstimator, TransformerMixin):
"""Kernel Principal component analysis (KPCA)
Non-linear dimensionality reduction through the use of kernels (see
:ref:`metrics`).
Parameters
----------
n_components: int or None
Number of components. If None, all non-zero components are kept.
kernel: "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed"
Kernel.
Default: "linear"
degree : int, default=3
Degree for poly kernels. Ignored by other kernels.
gamma : float, optional
Kernel coefficient for rbf and poly kernels. Default: 1/n_features.
Ignored by other kernels.
coef0 : float, optional
Independent term in poly and sigmoid kernels.
Ignored by other kernels.
kernel_params : mapping of string to any, optional
Parameters (keyword arguments) and values for kernel passed as
callable object. Ignored by other kernels.
alpha: int
Hyperparameter of the ridge regression that learns the
inverse transform (when fit_inverse_transform=True).
Default: 1.0
fit_inverse_transform: bool
Learn the inverse transform for non-precomputed kernels.
(i.e. learn to find the pre-image of a point)
Default: False
eigen_solver: string ['auto'|'dense'|'arpack']
Select eigensolver to use. If n_components is much less than
the number of training samples, arpack may be more efficient
than the dense eigensolver.
tol: float
convergence tolerance for arpack.
Default: 0 (optimal value will be chosen by arpack)
max_iter : int
maximum number of iterations for arpack
Default: None (optimal value will be chosen by arpack)
remove_zero_eig : boolean, default=True
If True, then all components with zero eigenvalues are removed, so
that the number of components in the output may be < n_components
(and sometimes even zero due to numerical instability).
When n_components is None, this parameter is ignored and components
with zero eigenvalues are removed regardless.
Attributes
----------
lambdas_ :
Eigenvalues of the centered kernel matrix
alphas_ :
Eigenvectors of the centered kernel matrix
evals_ : array[float], shape=(n_features)
All eigenvalues of centered kernel matrix
evecs_ : array[float, float], shape=(n_features, n_samples)
All eigenvectors of centered kernel matrix
dual_coef_ :
Inverse transform matrix
X_transformed_fit_ :
Projection of the fitted data on the kernel principal components
References
----------
Kernel PCA was introduced in:
Bernhard Schoelkopf, Alexander J. Smola,
and Klaus-Robert Mueller. 1999. Kernel principal
component analysis. In Advances in kernel methods,
MIT Press, Cambridge, MA, USA 327-352.
"""
def __init__(self,
n_components=None,
kernel="linear",
gamma=None,
degree=3,
coef0=1,
kernel_params=None,
alpha=1.0,
fit_inverse_transform=False,
eigen_solver='auto',
tol=0,
max_iter=None,
remove_zero_eig=False):
if fit_inverse_transform and kernel == 'precomputed':
raise ValueError(
"Cannot fit_inverse_transform with a precomputed kernel.")
self.n_components = n_components
self.kernel = kernel
self.kernel_params = kernel_params
self.gamma = gamma
self.degree = degree
self.coef0 = coef0
self.alpha = alpha
self.fit_inverse_transform = fit_inverse_transform
self.eigen_solver = eigen_solver
self.remove_zero_eig = remove_zero_eig
self.tol = tol
self.max_iter = max_iter
self._centerer = KernelCenterer()
@property
def _pairwise(self):
return self.kernel == "precomputed"
def _get_kernel(self, X, Y=None):
if callable(self.kernel):
params = self.kernel_params or {}
else:
params = {
"gamma": self.gamma,
"degree": self.degree,
"coef0": self.coef0
}
return pairwise_kernels(X,
Y,
metric=self.kernel,
filter_params=True,
**params)
def _fit_transform(self, K):
""" Fit's using kernel K"""
# center kernel
K = self._centerer.fit_transform(K)
if self.n_components is None:
n_components = K.shape[0]
else:
n_components = min(K.shape[0], self.n_components)
# compute eigenvectors
if self.eigen_solver == 'auto':
if K.shape[0] > 200 and n_components < 10:
eigen_solver = 'arpack'
else:
eigen_solver = 'dense'
else:
eigen_solver = self.eigen_solver
if eigen_solver == 'dense':
self.lambdas_, self.alphas_ = linalg.eigh(
K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1))
self.evals_, self.evecs_ = linalg.eigh(K)
elif eigen_solver == 'arpack':
self.lambdas_, self.alphas_ = eigsh(K,
n_components,
which="LA",
tol=self.tol,
maxiter=self.max_iter)
# sort eigenvectors in descending order
indices = self.lambdas_.argsort()[::-1]
self.lambdas_ = self.lambdas_[indices]
self.alphas_ = self.alphas_[:, indices]
# remove eigenvectors with a zero eigenvalue
if self.remove_zero_eig or self.n_components is None:
self.alphas_ = self.alphas_[:, self.lambdas_ > 0]
self.lambdas_ = self.lambdas_[self.lambdas_ > 0]
return K
def _fit_inverse_transform(self, X_transformed, X):
if hasattr(X, "tocsr"):
raise NotImplementedError("Inverse transform not implemented for "
"sparse matrices!")
n_samples = X_transformed.shape[0]
K = self._get_kernel(X_transformed)
K.flat[::n_samples + 1] += self.alpha
self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True)
self.X_transformed_fit_ = X_transformed
def fit(self, X, y=None):
"""Fit the model from data in X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
self : object
Returns the instance itself.
"""
K = self._get_kernel(X)
self._fit_transform(K)
if self.fit_inverse_transform:
sqrt_lambdas = np.diag(np.sqrt(self.lambdas_))
X_transformed = np.dot(self.alphas_, sqrt_lambdas)
self._fit_inverse_transform(X_transformed, X)
self.X_fit_ = X
return self
def fit_transform(self, X, y=None, **params):
"""Fit the model from data in X and transform X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Training vector, where n_samples in the number of samples
and n_features is the number of features.
Returns
-------
X_new: array-like, shape (n_samples, n_components)
"""
self.fit(X, **params)
X_transformed = self.alphas_ * np.sqrt(self.lambdas_)
if self.fit_inverse_transform:
self._fit_inverse_transform(X_transformed, X)
return X_transformed
def transform(self, X):
"""Transform X.
Parameters
----------
X: array-like, shape (n_samples, n_features)
Returns
-------
X_new: array-like, shape (n_samples, n_components)
"""
check_is_fitted(self, 'X_fit_')
K = self._centerer.transform(self._get_kernel(X, self.X_fit_))
return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_))
def inverse_transform(self, X):
"""Transform X back to original space.
Parameters
----------
X: array-like, shape (n_samples, n_components)
Returns
-------
X_new: array-like, shape (n_samples, n_features)
References
----------
"Learning to Find Pre-Images", G BakIr et al, 2004.
"""
if not self.fit_inverse_transform:
raise NotFittedError("The fit_inverse_transform parameter was not"
" set to True when instantiating and hence "
"the inverse transform is not available.")
K = self._get_kernel(X, self.X_transformed_fit_)
return np.dot(K, self.dual_coef_)
class KernelPCA(TransformerMixin, BaseEstimator):
def __init__(self,
kernel="linear",
gamma=None,
degree=3,
coef0=1,
kernel_params=None,
alpha=1.0,
fit_inverse_transform=False,
eigen_solver='auto',
tol=0,
max_iter=None,
remove_zero_eig=False,
random_state=None,
copy_X=True,
n_jobs=None):
if fit_inverse_transform and kernel == 'precomputed':
raise ValueError(
"Cannot fit_inverse_transform with a precomputed kernel.")
self.kernel = kernel
self.kernel_params = kernel_params
self.gamma = gamma
self.degree = degree
self.coef0 = coef0
self.alpha = alpha
self.fit_inverse_transform = fit_inverse_transform
self.eigen_solver = eigen_solver
self.remove_zero_eig = remove_zero_eig
self.tol = tol
self.max_iter = max_iter
self.random_state = random_state
self.n_jobs = n_jobs
self.copy_X = copy_X
@property
def _pairwise(self):
return self.kernel == "precomputed"
def _get_kernel(self, X, Y=None):
if callable(self.kernel):
params = self.kernel_params or {}
else:
params = {
"gamma": self.gamma,
"degree": self.degree,
"coef0": self.coef0
}
return pairwise_kernels(X,
Y,
metric=self.kernel,
filter_params=True,
n_jobs=self.n_jobs,
**params)
def _fit_transform(self, K):
""" Fit's using kernel K"""
# center kernel
K = self._centerer.fit_transform(K)
return K
def _fit_inverse_transform(self, X_transformed, X):
if hasattr(X, "tocsr"):
raise NotImplementedError("Inverse transform not implemented for "
"sparse matrices!")
n_samples = X_transformed.shape[0]
K = self._get_kernel(X_transformed)
K.flat[::n_samples + 1] += self.alpha
self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True)
self.X_transformed_fit_ = X_transformed
def fit(self, X, y=None):
X = check_array(X, accept_sparse='csr', copy=self.copy_X)
self._centerer = KernelCenterer()
K = self._get_kernel(X)
self._fit_transform(K)
if self.fit_inverse_transform:
# no need to use the kernel to transform X, use shortcut expression
X_transformed = self.alphas_ * np.sqrt(self.lambdas_)
self._fit_inverse_transform(X_transformed, X)
self.X_fit_ = X
return self
def fit_transform(self, X, y=None, **params):
self.fit(X, **params)
# no need to use the kernel to transform X, use shortcut expression
X_transformed = self.alphas_ * np.sqrt(self.lambdas_)
if self.fit_inverse_transform:
self._fit_inverse_transform(X_transformed, X)
return X_transformed
def transform(self, X):
check_is_fitted(self)
# Compute centered gram matrix between X and training data X_fit_
K = self._centerer.transform(self._get_kernel(X, self.X_fit_))
# scale eigenvectors (properly account for null-space for dot product)
non_zeros = np.flatnonzero(self.lambdas_)
scaled_alphas = np.zeros_like(self.alphas_)
scaled_alphas[:, non_zeros] = (self.alphas_[:, non_zeros] /
np.sqrt(self.lambdas_[non_zeros]))
# Project with a scalar product between K and the scaled eigenvectors
return np.dot(K, scaled_alphas)
def inverse_transform(self, X):
if not self.fit_inverse_transform:
raise NotFittedError("The fit_inverse_transform parameter was not"
" set to True when instantiating and hence "
"the inverse transform is not available.")
K = self._get_kernel(X, self.X_transformed_fit_)
return np.dot(K, self.dual_coef_)