def chooseKernel(data, kerneltype='euclidean'): r"""Kernalize data (uses sklearn) Parameters ========== data : array of shape (n_individuals, n_dimensions) Data matrix. kerneltype : {'euclidean', 'cosine', 'laplacian', 'polynomial_kernel', 'jaccard'}, optional Kernel type. Returns ======= array of shape (n_individuals, n_individuals) """ if kerneltype == 'euclidean': K = np.divide(1, (1+pairwise_distances(data, metric="euclidean"))) elif kerneltype == 'cosine': K = (pairwise.cosine_kernel(data)) elif kerneltype == 'laplacian': K = (pairwise.laplacian_kernel(data)) elif kerneltype == 'linear': K = (pairwise.linear_kernel(data)) elif kerneltype == 'polynomial_kernel': K = (pairwise.polynomial_kernel(data)) elif kerneltype == 'jaccard': K = 1-distance.cdist(data, data, metric='jaccard') scaler = KernelCenterer().fit(K) return(scaler.transform(K))
def test_kernelcenterer_vs_sklearn(): # Compare msmbuilder.preprocessing.KernelCenterer # with sklearn.preprocessing.KernelCenterer kernelcentererr = KernelCentererR() kernelcentererr.fit(np.concatenate(trajs)) kernelcenterer = KernelCenterer() kernelcenterer.fit(trajs) y_ref1 = kernelcentererr.transform(trajs[0]) y1 = kernelcenterer.transform(trajs)[0] np.testing.assert_array_almost_equal(y_ref1, y1)
def chooseKernel(data, kerneltype='euclidean'): if kerneltype == 'euclidean': K = np.divide(1, (1 + pairwise_distances(data, metric="euclidean"))) elif kerneltype == 'cosine': K = (pairwise.cosine_kernel(data)) elif kerneltype == 'laplacian': K = (pairwise.laplacian_kernel(data)) elif kerneltype == 'linear': K = (pairwise.linear_kernel(data)) elif kerneltype == 'polynomial_kernel': K = (pairwise.polynomial_kernel(data)) elif kerneltype == 'jaccard': K = 1 - distance.cdist(data, data, metric='jaccard') scaler = KernelCenterer().fit(K) return (scaler.transform(K))
def cv_mkl(kernel_list, labels, mkl, n_folds, dataset, data): n_sample, n_labels = labels.shape n_km = len(kernel_list) tags = np.loadtxt("../data/cv/"+data+".cv") for i in range(1,n_folds+1): print "Test fold %d" %i res_f = "../svm_result/weights/"+dataset+"_fold_%d_%s.weights" % (i,mkl) para_f = "../svm_result/upperbound/"+dataset+"_fold_%d_%s.ubound" % (i,mkl) test = np.array(tags == i) train = np.array(~test) train_y = labels[train,:] test_y = labels[test,:] n_train = len(train_y) n_test = len(test_y) train_km_list = [] # all train kernels are nomalized and centered for km in kernel_list: kc = KernelCenterer() train_km = km[np.ix_(train, train)] # center train and test kernels kc.fit(train_km) train_km_c = kc.transform(train_km) train_km_list.append(train_km_c) if mkl == 'UNIMKL': res = UNIMKL(train_km_list, train_y) np.savetxt(res_f, res) if mkl == 'ALIGNF2': res = alignf2(train_km_list, train_y, data) np.savetxt(res_f, res) if mkl.find('ALIGNF2SOFT') != -1: bestC, res = ALIGNF2SOFT(train_km_list, train_y, i, tags, data) np.savetxt(res_f, res) np.savetxt(para_f, bestC) if mkl == "TSMKL": W = np.zeros((n_km, n_labels)) for j in xrange(n_labels): print "..label",j W[:,j] = TSMKL(train_km_list, train_y[:,j]) res_f = "../svm_result/weights/"+dataset+"_fold_%d_%s.weights" % (i,mkl) np.savetxt(res_f, W)
class KernelPca: # beta: ガウスカーネルパラメータ def __init__(self, beta): self.beta = beta self.centerer = KernelCenterer() # gauss kernel def __kernel(self, x1, x2): return np.exp(-self.beta * np.linalg.norm(x1 - x2)**2) # データを入力して主成分ベクトルを計算する # shape(X) = (N, M) # n: 抽出する主成分の数 def fit_transform(self, X, n): self.X = X # グラム行列 N = X.shape[0] K = np.array([[self.__kernel(X[i], X[j]) for j in range(N)] for i in range(N)]) # 中心化 K = self.centerer.fit_transform(K) # eighは固有値の昇順で出力される vals, vecs = np.linalg.eigh(K) vals = vals[::-1] vecs = vecs[:, ::-1] # 特異値と左特異ベクトル、上位n個 self.sigma = np.sqrt(vals[:n]) # (n) self.a = np.array(vecs[:, :n]) # (N,n) return self.sigma * self.a # (N,n) # xの主成分表示を返す # shape(x)=(Nx, M) def transform(self, x): # グラム行列 N = self.X.shape[0] Nx = x.shape[0] K = np.array([[self.__kernel(x[i], self.X[j]) for j in range(N)] for i in range(Nx)]) # (Nx,N) # 中心化 K = self.centerer.transform(K) # 主成分を計算 return K.dot(self.a) / self.sigma # (Nx,n)
def test_center_kernel(): """Test that KernelCenterer is equivalent to Scaler in feature space""" X_fit = np.random.random((5, 4)) scaler = Scaler(with_std=False) scaler.fit(X_fit) X_fit_centered = scaler.transform(X_fit) K_fit = np.dot(X_fit, X_fit.T) # center fit time matrix centerer = KernelCenterer() K_fit_centered = np.dot(X_fit_centered, X_fit_centered.T) K_fit_centered2 = centerer.fit_transform(K_fit) assert_array_almost_equal(K_fit_centered, K_fit_centered2) # center predict time matrix X_pred = np.random.random((2, 4)) K_pred = np.dot(X_pred, X_fit.T) X_pred_centered = scaler.transform(X_pred) K_pred_centered = np.dot(X_pred_centered, X_fit_centered.T) K_pred_centered2 = centerer.transform(K_pred) assert_array_almost_equal(K_pred_centered, K_pred_centered2)
class kc(): def __init__(self, cols, metric): self.columns = cols self.metric = metric self.model = KernelCenterer() def fit(self, data): k = pairwise_kernels(data[self.columns], metric=self.metric) self.model.fit(k) def fit_transform(self, data): k = pairwise_kernels(data[self.columns], metric=self.metric) transformed = self.model.fit_transform(k) for idx in range(len(self.columns)): data[self.columns[idx]] = transformed[:, idx] return data def transform(self, data): k = pairwise_kernels(data[self.columns], metric=self.metric) transformed = self.model.transform(k) for idx in range(len(self.columns)): data[self.columns[idx]] = transformed[:, idx] return data
if __name__ == "__main__": classes = generate_spike_classes(1, 2) train = generate_spike_times(classes) test = generate_spike_times(classes) rasterPlot(train) K = compute_K_matrix(train) ############################### # N = K.shape[0] # H = np.eye(N) - np.tile(1./N, [N, N]); # Kc = np.dot(np.dot(H, K), H) kcenterer = KernelCenterer() # kcenterer.fit(K) # Center Kernel Matrix Kc = kcenterer.transform(K) # ############################### D, E = eig(Kc) proj = np.dot(Kc, E[:, 0:2]) ################################ Center test Kt = compute_K_matrix(train, test) # M = Kt.shape[0] # A = np.tile(K.sum(axis=0), [M, 1]) / N # B = np.tile(Kt.sum(axis=1),[N, 1]) /N # Kc2 = Kt - A - B + K.sum()/ N**2; Kc2 = kcenterer.transform(Kt) proj2 = np.dot(Kc2, E[:, 0:2]) # kpca = KernelPCA(kernel="precomputed", n_components=2) # kpca.fit(Kc)
if __name__ == '__main__': classes = generate_spike_classes(1, 2) train = generate_spike_times(classes) test = generate_spike_times(classes) rasterPlot(train) K = compute_K_matrix(train) ############################### #N = K.shape[0] #H = np.eye(N) - np.tile(1./N, [N, N]); #Kc = np.dot(np.dot(H, K), H) kcenterer = KernelCenterer() # kcenterer.fit(K) # Center Kernel Matrix Kc = kcenterer.transform(K) # ############################### D, E = eig(Kc) proj = np.dot(Kc, E[:, 0:2]) ################################ Center test Kt = compute_K_matrix(train, test) #M = Kt.shape[0] #A = np.tile(K.sum(axis=0), [M, 1]) / N #B = np.tile(Kt.sum(axis=1),[N, 1]) /N #Kc2 = Kt - A - B + K.sum()/ N**2; Kc2 = kcenterer.transform(Kt) proj2 = np.dot(Kc2, E[:, 0:2]) #kpca = KernelPCA(kernel="precomputed", n_components=2) #kpca.fit(Kc)
class KernelECA(BaseEstimator, TransformerMixin): """Kernel Entropy component analysis (KECA) Non-linear dimensionality reduction through the use of kernels (see :ref:`metrics`). Parameters ---------- n_components: int or None Number of components. If None, all non-zero components are kept. kernel: "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed" Kernel. Default: "linear" degree : int, default=3 Degree for poly kernels. Ignored by other kernels. gamma : float, optional Kernel coefficient for rbf and poly kernels. Default: 1/n_features. Ignored by other kernels. coef0 : float, optional Independent term in poly and sigmoid kernels. Ignored by other kernels. kernel_params : mapping of string to any, optional Parameters (keyword arguments) and values for kernel passed as callable object. Ignored by other kernels. eigen_solver: string ['auto'|'dense'|'arpack'] Select eigensolver to use. If n_components is much less than the number of training samples, arpack may be more efficient than the dense eigensolver. tol: float convergence tolerance for arpack. Default: 0 (optimal value will be chosen by arpack) max_iter : int maximum number of iterations for arpack Default: None (optimal value will be chosen by arpack) random_state : int seed, RandomState instance, or None, default : None A pseudo random number generator used for the initialization of the residuals when eigen_solver == 'arpack'. Attributes ---------- lambdas_ : Eigenvalues of the centered kernel matrix alphas_ : Eigenvectors of the centered kernel matrix dual_coef_ : Inverse transform matrix X_transformed_fit_ : Projection of the fitted data on the kernel entropy components References ---------- Kernel ECA based on: (c) Robert Jenssen, University of Tromso, Norway, 2010 R. Jenssen, "Kernel Entropy Component Analysis," IEEE Trans. Patt. Anal. Mach. Intel., 32(5), 847-860, 2010. """ def __init__(self, n_components=None, kernel="linear", gamma=None, degree=3, coef0=1, kernel_params=None, eigen_solver='auto', tol=0, max_iter=None, random_state=None,center=False): self.n_components = n_components self._kernel = kernel self.kernel_params = kernel_params self.gamma = gamma self.degree = degree self.coef0 = coef0 self.eigen_solver = eigen_solver self.tol = tol self.max_iter = max_iter self.random_state = random_state self._centerer = KernelCenterer() self.center = center @property def _pairwise(self): return self.kernel == "precomputed" def _get_kernel(self, X, Y=None): if callable(self._kernel): params = self.kernel_params or {} else: params = {"gamma": self.gamma, "degree": self.degree, "coef0": self.coef0} return pairwise_kernels(X, Y, metric=self._kernel, filter_params=True, **params) def _fit_transform(self, K): """ Fit's using kernel K""" # center kernel if self.center == True: K = self._centerer.fit_transform(K) X_transformed = self.kernelECA(K=K) self.X_transformed = X_transformed return K def fit(self, X, y=None): """Fit the model from data in X. Parameters ---------- X: array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the instance itself. """ K = self._get_kernel(X) self._fit_transform(K) self.X_fit_ = X return self def fit_transform(self, X, y=None, **params): """Fit the model from data in X and transform X. Parameters ---------- X: array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- X_new: array-like, shape (n_samples, n_components) """ self.fit(X, **params) X_transformed= self.X_transformed return X_transformed def transform(self, X): """Transform X. Parameters ---------- X: array-like, shape (n_samples, n_features) Returns ------- X_new: array-like, shape (n_samples, n_components) """ check_is_fitted(self, 'X_fit_') K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_)) def inverse_transform(self, X): raise NotImplementedError("Function inverse_transform is not implemented.") # here are the helper functions => to integrate in the code! def kernelECA(self,K): if self.n_components is None: n_components = K.shape[0] else: n_components = min(K.shape[0], self.n_components) # compute eigenvectors self.lambdas_, self.alphas_ = linalg.eigh(K) d = self.lambdas_ E = self.alphas_ # sort eigenvectors in descending order D,E = self.sort_eigenvalues(d,E) d = np.diag(D) sorted_entropy_index,entropy = self.ECA(D,E) Es = E[:,sorted_entropy_index] ds = d[sorted_entropy_index] Phi = np.zeros((K.shape[0],n_components)) for i in range(n_components): Phi[:,i] = np.sqrt(ds[i]) * Es[:,i] X_transformed = Phi return X_transformed def sort_eigenvalues(self,D,E): d = D indices = np.argsort(d)[::-1] d = d[indices] D = np.zeros((len(d),len(d))) for i in range(len(d)): D[i,i] = d[i] E = E[:,indices] return D,E def ECA(self,D,E): N = E.shape[0] entropy = np.multiply(np.diag(D).T , (np.dot(np.ones((1,N)),E))**2)[0] indices = np.argsort(entropy)[::-1] entropy = entropy[indices] return indices,entropy
class KernelECA(BaseEstimator, TransformerMixin): """Kernel Entropy component analysis (KECA) Non-linear dimensionality reduction through the use of kernels (see :ref:`metrics`). Parameters ---------- n_components: int or None Number of components. If None, all non-zero components are kept. kernel: "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed" Kernel. Default: "linear" degree : int, default=3 Degree for poly kernels. Ignored by other kernels. gamma : float, optional Kernel coefficient for rbf and poly kernels. Default: 1/n_features. Ignored by other kernels. coef0 : float, optional Independent term in poly and sigmoid kernels. Ignored by other kernels. kernel_params : mapping of string to any, optional Parameters (keyword arguments) and values for kernel passed as callable object. Ignored by other kernels. eigen_solver: string ['auto'|'dense'|'arpack'] Select eigensolver to use. If n_components is much less than the number of training samples, arpack may be more efficient than the dense eigensolver. tol: float convergence tolerance for arpack. Default: 0 (optimal value will be chosen by arpack) max_iter : int maximum number of iterations for arpack Default: None (optimal value will be chosen by arpack) random_state : int seed, RandomState instance, or None, default : None A pseudo random number generator used for the initialization of the residuals when eigen_solver == 'arpack'. Attributes ---------- lambdas_ : Eigenvalues of the centered kernel matrix alphas_ : Eigenvectors of the centered kernel matrix dual_coef_ : Inverse transform matrix X_transformed_fit_ : Projection of the fitted data on the kernel entropy components References ---------- Kernel ECA based on: (c) Robert Jenssen, University of Tromso, Norway, 2010 R. Jenssen, "Kernel Entropy Component Analysis," IEEE Trans. Patt. Anal. Mach. Intel., 32(5), 847-860, 2010. """ def __init__(self, n_components=None, kernel="linear", gamma=None, degree=3, coef0=1, kernel_params=None, eigen_solver='auto', tol=0, max_iter=None, random_state=None, center=False): self.n_components = n_components self._kernel = kernel self.kernel_params = kernel_params self.gamma = gamma self.degree = degree self.coef0 = coef0 self.eigen_solver = eigen_solver self.tol = tol self.max_iter = max_iter self.random_state = random_state self._centerer = KernelCenterer() self.center = center @property def _pairwise(self): return self.kernel == "precomputed" def _get_kernel(self, X, Y=None): if callable(self._kernel): params = self.kernel_params or {} else: params = { "gamma": self.gamma, "degree": self.degree, "coef0": self.coef0 } return pairwise_kernels(X, Y, metric=self._kernel, filter_params=True, **params) def _fit_transform(self, K): """ Fit's using kernel K""" # center kernel if self.center == True: K = self._centerer.fit_transform(K) X_transformed = self.kernelECA(K=K) self.X_transformed = X_transformed return K def fit(self, X, y=None): """Fit the model from data in X. Parameters ---------- X: array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the instance itself. """ K = self._get_kernel(X) self._fit_transform(K) self.X_fit_ = X return self def fit_transform(self, X, y=None, **params): """Fit the model from data in X and transform X. Parameters ---------- X: array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- X_new: array-like, shape (n_samples, n_components) """ self.fit(X, **params) X_transformed = self.X_transformed return X_transformed def transform(self, X): """Transform X. Parameters ---------- X: array-like, shape (n_samples, n_features) Returns ------- X_new: array-like, shape (n_samples, n_components) """ check_is_fitted(self, 'X_fit_') K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_)) def inverse_transform(self, X): raise NotImplementedError( "Function inverse_transform is not implemented.") # here are the helper functions => to integrate in the code! def kernelECA(self, K): if self.n_components is None: n_components = K.shape[0] else: n_components = min(K.shape[0], self.n_components) # compute eigenvectors self.lambdas_, self.alphas_ = linalg.eigh(K) d = self.lambdas_ E = self.alphas_ # sort eigenvectors in descending order D, E = self.sort_eigenvalues(d, E) d = np.diag(D) sorted_entropy_index, entropy = self.ECA(D, E) Es = E[:, sorted_entropy_index] ds = d[sorted_entropy_index] Phi = np.zeros((K.shape[0], n_components)) for i in range(n_components): Phi[:, i] = np.sqrt(ds[i]) * Es[:, i] X_transformed = Phi return X_transformed def sort_eigenvalues(self, D, E): d = D indices = np.argsort(d)[::-1] d = d[indices] D = np.zeros((len(d), len(d))) for i in range(len(d)): D[i, i] = d[i] E = E[:, indices] return D, E def ECA(self, D, E): N = E.shape[0] entropy = np.multiply(np.diag(D).T, (np.dot(np.ones((1, N)), E))**2)[0] indices = np.argsort(entropy)[::-1] entropy = entropy[indices] return indices, entropy
class MIDA(BaseEstimator, TransformerMixin): """Maximum independence domain adaptation Args: n_components (int): Number of components to keep. kernel (str): "linear", "rbf", or "poly". Kernel to use for MIDA. Defaults to "linear". mu (float): Hyperparameter of the l2 penalty. Defaults to 1.0. eta (float): Hyperparameter of the label dependence. Defaults to 1.0. augmentation (bool): Whether using covariates as augment features. Defaults to False. kernel_params (dict or None): Parameters for the kernel. Defaults to None. References: [1] Yan, K., Kou, L. and Zhang, D., 2018. Learning domain-invariant subspace using domain features and independence maximization. IEEE transactions on cybernetics, 48(1), pp.288-299. """ def __init__( self, n_components, kernel="linear", lambda_=1.0, mu=1.0, eta=1.0, augmentation=False, kernel_params=None, ): self.n_components = n_components self.kernel = kernel self.mu = mu self.eta = eta self.augmentation = augmentation if kernel_params is None: self.kernel_params = {} else: self.kernel_params = kernel_params self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1) self._centerer = KernelCenterer() self.x_fit = None def _get_kernel(self, x, y=None): if self.kernel in ["linear", "rbf", "poly"]: params = self.kernel_params or {} else: raise ValueError("Pre-computed kernel not supported") return pairwise_kernels(x, y, metric=self.kernel, filter_params=True, **params) def fit(self, x, y=None, covariates=None): """ Args: x : array-like. Input data, shape (n_samples, n_features) y : array-like. Labels, shape (nl_samples,) covariates : array-like. Domain co-variates, shape (n_samples, n_co-variates) Note: Unsupervised MIDA is performed if y is None. Semi-supervised MIDA is performed is y is not None. """ if self.augmentation and type(covariates) == np.ndarray: x = np.concatenate((x, covariates), axis=1) # Kernel matrix kernel_x = self._get_kernel(x) kernel_x[np.isnan(kernel_x)] = 0 # Solve the optimization problem self._fit(kernel_x, y, covariates) self.x_fit = x return self def _fit(self, kernel_x, y, covariates=None): """solve MIDA Args: kernel_x: array-like, kernel matrix of input data x, shape (n_samples, n_samples) y: array-like. Labels, shape (nl_samples,) covariates: array-like. Domain co-variates, shape (n_samples, n_covariates) Returns: self """ n_samples = kernel_x.shape[0] # Identity (unit) matrix unit_mat = np.eye(n_samples) # Centering matrix ctr_mat = unit_mat - 1.0 / n_samples * np.ones((n_samples, n_samples)) kernel_x = self._centerer.fit_transform(kernel_x) if type(covariates) == np.ndarray: kernel_c = np.dot(covariates, covariates.T) else: kernel_c = np.zeros((n_samples, n_samples)) if y is not None: n_labeled = y.shape[0] if n_labeled > n_samples: raise ValueError("Number of labels exceeds number of samples") y_mat_ = self._label_binarizer.fit_transform(y) y_mat = np.zeros((n_samples, y_mat_.shape[1])) y_mat[:n_labeled, :] = y_mat_ ker_y = np.dot(y_mat, y_mat.T) obj = multi_dot([ kernel_x, self.mu * ctr_mat + self.eta * multi_dot([ctr_mat, ker_y, ctr_mat]) - multi_dot([ctr_mat, kernel_c, ctr_mat]), kernel_x.T, ]) else: obj = multi_dot([ kernel_x, self.mu * ctr_mat - multi_dot([ctr_mat, kernel_c, ctr_mat]), kernel_x.T ]) eig_values, eig_vectors = linalg.eigh( obj, subset_by_index=[n_samples - self.n_components, n_samples - 1]) idx_sorted = eig_values.argsort()[::-1] self.eig_values_ = eig_values[idx_sorted] self.U = eig_vectors[:, idx_sorted] self.U = np.asarray(self.U, dtype=np.float) return self def fit_transform(self, x, y=None, covariates=None): """ Args: x : array-like, shape (n_samples, n_features) y : array-like, shape (n_samples,) covariates : array-like, shape (n_samples, n_covariates) Returns: x_transformed : array-like, shape (n_samples, n_components) """ self.fit(x, y, covariates) return self.transform(x, covariates) def transform(self, x, covariates=None): """ Args: x : array-like, shape (n_samples, n_features) covariates : array-like, augmentation features, shape (n_samples, n_covariates) Returns: x_transformed : array-like, shape (n_samples, n_components) """ check_is_fitted(self, "x_fit") if type(covariates) == np.ndarray and self.augmentation: x = np.concatenate((x, covariates), axis=1) kernel_x = self._centerer.transform( pairwise_kernels(x, self.x_fit, metric=self.kernel, filter_params=True, **self.kernel_params)) return np.dot(kernel_x, self.U)
class KernelPCA(BaseEstimator, TransformerMixin): """Kernel Principal component analysis (KPCA) Non-linear dimensionality reduction through the use of kernels (see :ref:`metrics`). Read more in the :ref:`User Guide <kernel_PCA>`. Parameters ---------- n_components : int, default=None Number of components. If None, all non-zero components are kept. kernel : "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed" Kernel. Default="linear". gamma : float, default=1/n_features Kernel coefficient for rbf, poly and sigmoid kernels. Ignored by other kernels. degree : int, default=3 Degree for poly kernels. Ignored by other kernels. coef0 : float, default=1 Independent term in poly and sigmoid kernels. Ignored by other kernels. kernel_params : mapping of string to any, default=None Parameters (keyword arguments) and values for kernel passed as callable object. Ignored by other kernels. alpha : int, default=1.0 Hyperparameter of the ridge regression that learns the inverse transform (when fit_inverse_transform=True). fit_inverse_transform : bool, default=False Learn the inverse transform for non-precomputed kernels. (i.e. learn to find the pre-image of a point) eigen_solver : string ['auto'|'dense'|'arpack'], default='auto' Select eigensolver to use. If n_components is much less than the number of training samples, arpack may be more efficient than the dense eigensolver. tol : float, default=0 Convergence tolerance for arpack. If 0, optimal value will be chosen by arpack. max_iter : int, default=None Maximum number of iterations for arpack. If None, optimal value will be chosen by arpack. remove_zero_eig : boolean, default=False If True, then all components with zero eigenvalues are removed, so that the number of components in the output may be < n_components (and sometimes even zero due to numerical instability). When n_components is None, this parameter is ignored and components with zero eigenvalues are removed regardless. random_state : int, RandomState instance or None, optional (default=None) If int, random_state is the seed used by the random number generator; If RandomState instance, random_state is the random number generator; If None, the random number generator is the RandomState instance used by `np.random`. Used when ``eigen_solver`` == 'arpack'. .. versionadded:: 0.18 copy_X : boolean, default=True If True, input X is copied and stored by the model in the `X_fit_` attribute. If no further changes will be done to X, setting `copy_X=False` saves memory by storing a reference. .. versionadded:: 0.18 n_jobs : int, default=1 The number of parallel jobs to run. If `-1`, then the number of jobs is set to the number of CPU cores. .. versionadded:: 0.18 Attributes ---------- lambdas_ : array, (n_components,) Eigenvalues of the centered kernel matrix in decreasing order. If `n_components` and `remove_zero_eig` are not set, then all values are stored. alphas_ : array, (n_samples, n_components) Eigenvectors of the centered kernel matrix. If `n_components` and `remove_zero_eig` are not set, then all components are stored. dual_coef_ : array, (n_samples, n_features) Inverse transform matrix. Set if `fit_inverse_transform` is True. X_transformed_fit_ : array, (n_samples, n_components) Projection of the fitted data on the kernel principal components. X_fit_ : (n_samples, n_features) The data used to fit the model. If `copy_X=False`, then `X_fit_` is a reference. This attribute is used for the calls to transform. References ---------- Kernel PCA was introduced in: Bernhard Schoelkopf, Alexander J. Smola, and Klaus-Robert Mueller. 1999. Kernel principal component analysis. In Advances in kernel methods, MIT Press, Cambridge, MA, USA 327-352. """ def __init__(self, n_components=None, kernel="linear", gamma=None, degree=3, coef0=1, kernel_params=None, alpha=1.0, fit_inverse_transform=False, eigen_solver='auto', tol=0, max_iter=None, remove_zero_eig=False, random_state=None, copy_X=True, n_jobs=1): if fit_inverse_transform and kernel == 'precomputed': raise ValueError( "Cannot fit_inverse_transform with a precomputed kernel.") self.n_components = n_components self.kernel = kernel self.kernel_params = kernel_params self.gamma = gamma self.degree = degree self.coef0 = coef0 self.alpha = alpha self.fit_inverse_transform = fit_inverse_transform self.eigen_solver = eigen_solver self.remove_zero_eig = remove_zero_eig self.tol = tol self.max_iter = max_iter self._centerer = KernelCenterer() self.random_state = random_state self.n_jobs = n_jobs self.copy_X = copy_X @property def _pairwise(self): return self.kernel == "precomputed" def _get_kernel(self, X, Y=None): if callable(self.kernel): params = self.kernel_params or {} else: params = { "gamma": self.gamma, "degree": self.degree, "coef0": self.coef0 } return pairwise_kernels(X, Y, metric=self.kernel, filter_params=True, n_jobs=self.n_jobs, **params) def _fit_transform(self, K): """ Fit's using kernel K""" # center kernel K = self._centerer.fit_transform(K) if self.n_components is None: n_components = K.shape[0] else: n_components = min(K.shape[0], self.n_components) # compute eigenvectors if self.eigen_solver == 'auto': if K.shape[0] > 200 and n_components < 10: eigen_solver = 'arpack' else: eigen_solver = 'dense' else: eigen_solver = self.eigen_solver if eigen_solver == 'dense': self.lambdas_, self.alphas_ = linalg.eigh( K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1)) elif eigen_solver == 'arpack': random_state = check_random_state(self.random_state) # initialize with [-1,1] as in ARPACK v0 = random_state.uniform(-1, 1, K.shape[0]) self.lambdas_, self.alphas_ = eigsh(K, n_components, which="LA", tol=self.tol, maxiter=self.max_iter, v0=v0) # sort eigenvectors in descending order indices = self.lambdas_.argsort()[::-1] self.lambdas_ = self.lambdas_[indices] self.alphas_ = self.alphas_[:, indices] # remove eigenvectors with a zero eigenvalue if self.remove_zero_eig or self.n_components is None: self.alphas_ = self.alphas_[:, self.lambdas_ > 0] self.lambdas_ = self.lambdas_[self.lambdas_ > 0] return K def _fit_inverse_transform(self, X_transformed, X): if hasattr(X, "tocsr"): raise NotImplementedError("Inverse transform not implemented for " "sparse matrices!") n_samples = X_transformed.shape[0] K = self._get_kernel(X_transformed) K.flat[::n_samples + 1] += self.alpha self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True) self.X_transformed_fit_ = X_transformed def fit(self, X, y=None): """Fit the model from data in X. Parameters ---------- X : array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the instance itself. """ X = check_array(X, accept_sparse='csr', copy=self.copy_X) K = self._get_kernel(X) self._fit_transform(K) if self.fit_inverse_transform: sqrt_lambdas = np.diag(np.sqrt(self.lambdas_)) X_transformed = np.dot(self.alphas_, sqrt_lambdas) self._fit_inverse_transform(X_transformed, X) self.X_fit_ = X return self def fit_transform(self, X, y=None, **params): """Fit the model from data in X and transform X. Parameters ---------- X : array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- X_new : array-like, shape (n_samples, n_components) """ self.fit(X, **params) X_transformed = self.alphas_ * np.sqrt(self.lambdas_) if self.fit_inverse_transform: self._fit_inverse_transform(X_transformed, X) return self.alphas_, self.lambdas_ def transform(self, X): """Transform X. Parameters ---------- X : array-like, shape (n_samples, n_features) Returns ------- X_new : array-like, shape (n_samples, n_components) """ check_is_fitted(self, 'X_fit_') K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_)) def get_eigen(self): return self.alphas_, self.lambdas_ def inverse_transform(self, X): """Transform X back to original space. Parameters ---------- X : array-like, shape (n_samples, n_components) Returns ------- X_new : array-like, shape (n_samples, n_features) References ---------- "Learning to Find Pre-Images", G BakIr et al, 2004. """ if not self.fit_inverse_transform: raise NotFittedError("The fit_inverse_transform parameter was not" " set to True when instantiating and hence " "the inverse transform is not available.") K = self._get_kernel(X, self.X_transformed_fit_) return np.dot(K, self.dual_coef_)
class KernelPCA(TransformerMixin, BaseEstimator): """Kernel Principal component analysis (KPCA) Non-linear dimensionality reduction through the use of kernels (see :ref:`metrics`). Read more in the :ref:`User Guide <kernel_PCA>`. Parameters ---------- n_components : int, default=None Number of components. If None, all non-zero components are kept. kernel : "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed" Kernel. Default="linear". gamma : float, default=1/n_features Kernel coefficient for rbf, poly and sigmoid kernels. Ignored by other kernels. degree : int, default=3 Degree for poly kernels. Ignored by other kernels. coef0 : float, default=1 Independent term in poly and sigmoid kernels. Ignored by other kernels. kernel_params : mapping of string to any, default=None Parameters (keyword arguments) and values for kernel passed as callable object. Ignored by other kernels. alpha : int, default=1.0 Hyperparameter of the ridge regression that learns the inverse transform (when fit_inverse_transform=True). fit_inverse_transform : bool, default=False Learn the inverse transform for non-precomputed kernels. (i.e. learn to find the pre-image of a point) eigen_solver : string ['auto'|'dense'|'arpack'], default='auto' Select eigensolver to use. If n_components is much less than the number of training samples, arpack may be more efficient than the dense eigensolver. tol : float, default=0 Convergence tolerance for arpack. If 0, optimal value will be chosen by arpack. max_iter : int, default=None Maximum number of iterations for arpack. If None, optimal value will be chosen by arpack. remove_zero_eig : boolean, default=False If True, then all components with zero eigenvalues are removed, so that the number of components in the output may be < n_components (and sometimes even zero due to numerical instability). When n_components is None, this parameter is ignored and components with zero eigenvalues are removed regardless. random_state : int, RandomState instance or None, optional (default=None) If int, random_state is the seed used by the random number generator; If RandomState instance, random_state is the random number generator; If None, the random number generator is the RandomState instance used by `np.random`. Used when ``eigen_solver`` == 'arpack'. sklearn. versionadded:: 0.18 copy_X : boolean, default=True If True, input X is copied and stored by the model in the `X_fit_` attribute. If no further changes will be done to X, setting `copy_X=False` saves memory by storing a reference. sklearn. versionadded:: 0.18 n_jobs : int or None, optional (default=None) The number of parallel jobs to run. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary <n_jobs>` for more details. sklearn. versionadded:: 0.18 Attributes ---------- lambdas_ : array, (n_components,) Eigenvalues of the centered kernel matrix in decreasing order. If `n_components` and `remove_zero_eig` are not set, then all values are stored. alphas_ : array, (n_samples, n_components) Eigenvectors of the centered kernel matrix. If `n_components` and `remove_zero_eig` are not set, then all components are stored. dual_coef_ : array, (n_samples, n_features) Inverse transform matrix. Only available when ``fit_inverse_transform`` is True. X_transformed_fit_ : array, (n_samples, n_components) Projection of the fitted data on the kernel principal components. Only available when ``fit_inverse_transform`` is True. X_fit_ : (n_samples, n_features) The data used to fit the model. If `copy_X=False`, then `X_fit_` is a reference. This attribute is used for the calls to transform. Examples -------- >>> from sklearn.datasets import load_digits >>> from sklearn.decomposition import KernelPCA >>> X, _ = load_digits(return_X_y=True) >>> transformer = KernelPCA(n_components=7, kernel='linear') >>> X_transformed = transformer.fit_transform(X) >>> X_transformed.shape (1797, 7) References ---------- Kernel PCA was introduced in: Bernhard Schoelkopf, Alexander J. Smola, and Klaus-Robert Mueller. 1999. Kernel principal component analysis. In Advances in kernel methods, MIT Press, Cambridge, MA, USA 327-352. """ def __init__(self, n_components=None, kernel="linear", gamma=None, degree=3, coef0=1, kernel_params=None, alpha=1.0, fit_inverse_transform=False, eigen_solver='auto', tol=0, max_iter=None, remove_zero_eig=False, random_state=None, copy_X=True, n_jobs=None): if fit_inverse_transform and kernel == 'precomputed': raise ValueError( "Cannot fit_inverse_transform with a precomputed kernel.") self.n_components = n_components self.kernel = kernel self.kernel_params = kernel_params self.gamma = gamma self.degree = degree self.coef0 = coef0 self.alpha = alpha self.fit_inverse_transform = fit_inverse_transform self.eigen_solver = eigen_solver self.remove_zero_eig = remove_zero_eig self.tol = tol self.max_iter = max_iter self.random_state = random_state self.n_jobs = n_jobs self.copy_X = copy_X @property def _pairwise(self): return self.kernel == "precomputed" def _get_kernel(self, X, Y=None): if callable(self.kernel): params = self.kernel_params or {} else: params = { "gamma": self.gamma, "degree": self.degree, "coef0": self.coef0 } return pairwise_kernels(X, Y, metric=self.kernel, filter_params=True, n_jobs=self.n_jobs, **params) def _fit_transform(self, K): """ Fit's using kernel K""" # center kernel K = self._centerer.fit_transform(K) if self.n_components is None: n_components = K.shape[0] else: n_components = min(K.shape[0], self.n_components) # compute eigenvectors if self.eigen_solver == 'auto': if K.shape[0] > 200 and n_components < 10: eigen_solver = 'arpack' else: eigen_solver = 'dense' else: eigen_solver = self.eigen_solver if eigen_solver == 'dense': self.lambdas_, self.alphas_ = linalg.eigh( K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1)) elif eigen_solver == 'arpack': random_state = check_random_state(self.random_state) # initialize with [-1,1] as in ARPACK v0 = random_state.uniform(-1, 1, K.shape[0]) self.lambdas_, self.alphas_ = eigsh(K, n_components, which="LA", tol=self.tol, maxiter=self.max_iter, v0=v0) # make sure that the eigenvalues are ok and fix numerical issues self.lambdas_ = _check_psd_eigenvalues(self.lambdas_, enable_warnings=False) # flip eigenvectors' sign to enforce deterministic output self.alphas_, _ = svd_flip(self.alphas_, np.empty_like(self.alphas_).T) # sort eigenvectors in descending order indices = self.lambdas_.argsort()[::-1] self.lambdas_ = self.lambdas_[indices] self.alphas_ = self.alphas_[:, indices] # remove eigenvectors with a zero eigenvalue (null space) if required if self.remove_zero_eig or self.n_components is None: self.alphas_ = self.alphas_[:, self.lambdas_ > 0] self.lambdas_ = self.lambdas_[self.lambdas_ > 0] # Maintenance note on Eigenvectors normalization # ---------------------------------------------- # there is a link between # the eigenvectors of K=Phi(X)'Phi(X) and the ones of Phi(X)Phi(X)' # if v is an eigenvector of K # then Phi(X)v is an eigenvector of Phi(X)Phi(X)' # if u is an eigenvector of Phi(X)Phi(X)' # then Phi(X)'u is an eigenvector of Phi(X)Phi(X)' # # At this stage our self.alphas_ (the v) have norm 1, we need to scale # them so that eigenvectors in kernel feature space (the u) have norm=1 # instead # # We COULD scale them here: # self.alphas_ = self.alphas_ / np.sqrt(self.lambdas_) # # But choose to perform that LATER when needed, in `fit()` and in # `transform()`. return K def _fit_inverse_transform(self, X_transformed, X): if hasattr(X, "tocsr"): raise NotImplementedError("Inverse transform not implemented for " "sparse matrices!") n_samples = X_transformed.shape[0] K = self._get_kernel(X_transformed) K.flat[::n_samples + 1] += self.alpha self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True) self.X_transformed_fit_ = X_transformed def fit(self, X, y=None): """Fit the model from data in X. Parameters ---------- X : array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the instance itself. """ X = check_array(X, accept_sparse='csr', copy=self.copy_X) self._centerer = KernelCenterer() K = self._get_kernel(X) self.K = K self._fit_transform(K) if self.fit_inverse_transform: # no need to use the kernel to transform X, use shortcut expression X_transformed = self.alphas_ * np.sqrt(self.lambdas_) self._fit_inverse_transform(X_transformed, X) self.X_fit_ = X return self def fit_transform(self, X, y=None, **params): """Fit the model from data in X and transform X. Parameters ---------- X : array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- X_new : array-like, shape (n_samples, n_components) """ self.fit(X, **params) # no need to use the kernel to transform X, use shortcut expression X_transformed = self.alphas_ * np.sqrt(self.lambdas_) if self.fit_inverse_transform: self._fit_inverse_transform(X_transformed, X) return X_transformed def transform(self, X): """Transform X. Parameters ---------- X : array-like, shape (n_samples, n_features) Returns ------- X_new : array-like, shape (n_samples, n_components) """ check_is_fitted(self) # Compute centered gram matrix between X and training data X_fit_ K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) # scale eigenvectors (properly account for null-space for dot product) non_zeros = np.flatnonzero(self.lambdas_) scaled_alphas = np.zeros_like(self.alphas_) scaled_alphas[:, non_zeros] = (self.alphas_[:, non_zeros] / np.sqrt(self.lambdas_[non_zeros])) # Project with a scalar product between K and the scaled eigenvectors return np.dot(K, scaled_alphas) def inverse_transform(self, X): """Transform X back to original space. Parameters ---------- X : array-like, shape (n_samples, n_components) Returns ------- X_new : array-like, shape (n_samples, n_features) References ---------- "Learning to Find Pre-Images", G BakIr et al, 2004. """ if not self.fit_inverse_transform: raise NotFittedError("The fit_inverse_transform parameter was not" " set to True when instantiating and hence " "the inverse transform is not available.") K = self._get_kernel(X, self.X_transformed_fit_) return np.dot(K, self.dual_coef_)
def ovkr_mkl(kernel_list, labels, mkl, n_folds, dataset, data): n_sample, n_labels = labels.shape n_km = len(kernel_list) tags = np.loadtxt("../data/cv/"+data+".cv") #tags = np.array(range(n_sample)) % n_folds + 1 #np.random.seed(1234) #np.random.shuffle(tags) pred = np.zeros((n_sample, n_labels)) # Run for each fold for i in range(1,n_folds+1): print "Test fold %d" %i res_f = "../ovkr_result/weights/"+dataset+"_fold_%d_%s.weights" % (i,mkl) # divide data test = np.array(tags == i) train = np.array(~test) train_y = labels[train,:] test_y = labels[test,:] n_train = len(train_y) n_test = len(test_y) train_km_list = [] test_km_list = [] for km in kernel_list: kc = KernelCenterer() train_km = km[np.ix_(train, train)] test_km = km[np.ix_(test, train)] # center train and test kernels kc.fit(train_km) train_km_c = kc.transform(train_km) test_km_c = kc.transform(test_km) train_km_list.append(train_km_c) test_km_list.append(test_km_c) if mkl == 'UNIMKL': wei = UNIMKL(n_km, n_labels) else: wei = np.loadtxt(res_f, ndmin=2) normw = np.linalg.norm(wei) uni = np.ones(n_km) / np.linalg.norm(np.ones(n_km)) if normw == 0: wei[:,0] = uni else: wei[:,0] = wei[:,0] / normw train_ckm = np.zeros((n_train,n_train)) for t in range(n_km): train_ckm += wei[t,0]*train_km_list[t] # combine train and test kernel using learned weights test_ckm = np.zeros(test_km_list[0].shape) for t in range(n_km): test_ckm = test_ckm + wei[t,0]*test_km_list[t] AP = OVKR_train_CV(train_ckm, train_y, tags[train]) pred_label = OVKR_test(test_ckm, AP) pred[test, :] = pred_label return pred
def svm_mkl(kernel_list, labels, mkl, n_folds, dataset, data): n_sample, n_labels = labels.shape n_km = len(kernel_list) tags = np.loadtxt("../data/cv/"+data+".cv") pred = np.zeros((n_sample, n_labels)) # Run for each fold for i in range(1,n_folds+1): print "Test fold %d" %i res_f = "../svm_result/weights/"+dataset+"_fold_%d_%s.weights" % (i,mkl) # divide data test = np.array(tags == (i+1 if i+1<6 else 1)) train = np.array(~test) train_y = labels[train,:] test_y = labels[test,:] n_train = len(train_y) n_test = len(test_y) train_km_list = [] test_km_list = [] for km in kernel_list: kc = KernelCenterer() train_km = km[np.ix_(train, train)] test_km = km[np.ix_(test, train)] # center train and test kernels kc.fit(train_km) train_km_c = kc.transform(train_km) test_km_c = kc.transform(test_km) train_km_list.append(train_km_c) test_km_list.append(test_km_c) if mkl == 'UNIMKL': wei = UNIMKL(n_km, n_labels) else: wei = np.loadtxt(res_f, ndmin=2) # Normalized weights normw = np.linalg.norm(wei, 2, 0) uni = np.ones(n_km) / np.linalg.norm(np.ones(n_km)) for t in xrange(n_labels): if normw[t] == 0: # collapsed solution wei[:,t] = uni else: wei[:,t] = wei[:,t] / normw[t] for j in range(n_labels): tr_y = train_y[:,j] te_y = test_y[:,j] if wei.shape[1] == 1: wj = wei[:,0] else: wj = wei[:,j] ckm = np.zeros((n_train,n_train)) for t in range(n_km): ckm = ckm + wj[t]*train_km_list[t] # combine train and test kernel using learned weights train_ckm = ckm test_ckm = np.zeros(test_km_list[0].shape) for t in range(n_km): test_ckm = test_ckm + wj[t]*test_km_list[t] pred_label = svm(train_ckm, test_ckm, tr_y, te_y, tags[train], i) pred[test, j] = pred_label return pred
def cls(mkl): for data in datasets: print "####################" print '# ',data print "####################" # consider labels with more than 2% t = 0.02 datadir = '../data/' km_dir = datadir + data + "/" if data == 'Fingerprint': kernels = ['PPKr', 'NB','CP2','NI','LB','CPC','RLB','LC','LI','CPK','RLI','CSC'] km_list = [] y = np.loadtxt(km_dir+"y.txt",ndmin=2) p = np.sum(y==1,0)/float(y.shape[0]) y = y[:,p>t] for k in kernels: km_f = datadir + data + ("/%s.txt" % k) km_list.append(normalize_km(np.loadtxt(km_f))) pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl) pred = svm_mkl(km_list, y, mkl, 5, data,data) np.savetxt(pred_f, pred, fmt="%d") elif data == 'plant' or data == 'psortPos' or data == 'psortNeg': y = loadmat(km_dir+"label_%s.mat" % data)['y'].ravel() km_list = [] fs = commands.getoutput('ls %skern\;substr*.mat' % km_dir).split("\n") for f in fs: km = loadmat(f) km_list.append(km['K']) fs = commands.getoutput('ls %skern\;phylpro*.mat' % km_dir).split("\n") for f in fs: km = loadmat(f) km_list.append(km['K']) fs = commands.getoutput('ls %skm_evalue*.mat' % km_dir).split("\n") for f in fs: km = loadmat(f) km_list.append(km['K']) n_samples = y.shape[0] n_km = len(km_list) y_pred = np.zeros(n_samples) n_labels = 1 tags = np.loadtxt("../data/cv/"+data+".cv") for fold in range(1,6): test_ind = np.where(tags == fold)[0] train_ind = np.where(tags != fold)[0] train_km_list = [] test_km_list = [] train_y = y[train_ind] test_y = y[test_ind] n_train = len(train_ind) n_test = len(test_ind) w_f = "../svm_result/weights/"+data+"_fold_%d_%s.weights" % (fold,mkl) if mkl == 'UNIMKL': w = UNIMKL(n_km, n_labels).ravel() else: w = np.loadtxt(w_f, ndmin=2).ravel() normw = np.linalg.norm(w, 2, 0) uni = np.ones(n_km) / np.linalg.norm(np.ones(n_km)) if normw == 0: w = uni else: w = w / normw for km in km_list: kc = KernelCenterer() train_km = km[np.ix_(train_ind, train_ind)] test_km = km[np.ix_(test_ind, train_ind)] # center train and test kernels kc.fit(train_km) train_km_c = kc.transform(train_km) test_km_c = kc.transform(test_km) train_km_list.append(train_km_c) test_km_list.append(test_km_c) train_ckm = np.zeros((n_train,n_train)) for t in range(n_km): train_ckm = train_ckm + w[t]*train_km_list[t] test_ckm = np.zeros(test_km_list[0].shape) for t in range(n_km): test_ckm = test_ckm + w[t]*test_km_list[t] C_range = [0.01,0.1,1,10,100] param_grid = dict(C=C_range) cv = StratifiedShuffleSplit(train_y,n_iter=5,test_size=0.2,random_state=42) grid = GridSearchCV(SVC(kernel='precomputed'), param_grid=param_grid, cv=cv) grid.fit(train_ckm, train_y) bestC = grid.best_params_['C'] svm = SVC(kernel='precomputed', C=bestC) svm.fit(train_ckm, train_y) y_pred[test_ind] = svm.predict(test_ckm) pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl) np.savetxt(pred_f, y_pred, fmt="%d") elif data in image_datasets: y = np.loadtxt(km_dir+"y.txt",ndmin=2) p = np.sum(y==1,0)/float(y.shape[0]) y = y[:,p>t] linear_km_list = [] for i in range(1,16): name = 'kernel_linear_%d.txt' % i km_f = km_dir+name km = np.loadtxt(km_f) # normalize input kernel !!!!!!!! linear_km_list.append(normalize_km(km)) pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl) pred = svm_mkl(linear_km_list, y, mkl, 5, data,data) np.savetxt(pred_f, pred, fmt="%d") elif data == 'SPAMBASE': y = np.loadtxt(km_dir+"y.txt",ndmin=2) rbf_km_list = [] gammas = [2**-9, 2**-8, 2**-7, 2**-6, 2**-5, 2**-4, 2**-3] X = np.loadtxt(km_dir+"x.txt") scaler = preprocessing.StandardScaler().fit(X) X = scaler.transform(X) X = preprocessing.normalize(X) for gamma in gammas: km = rbf_kernel(X, gamma=gamma) rbf_km_list.append(km) pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl) pred = svm_mkl(rbf_km_list, y, mkl, 5, data,data) np.savetxt(pred_f, pred, fmt="%d") else: rbf_km_list = [] gammas = [2**-13,2**-11,2**-9,2**-7,2**-5,2**-3,2**-1,2**1,2**3] X = np.loadtxt(km_dir+"x.txt") scaler = preprocessing.StandardScaler().fit(X) X = scaler.transform(X) X = preprocessing.normalize(X) y = np.loadtxt(km_dir+"y.txt") p = np.sum(y==1,0)/float(y.shape[0]) y = y[:,p>t] for gamma in gammas: km = rbf_kernel(X, gamma=gamma) # normalize input kernel !!!!!!!! rbf_km_list.append(km) pred_f = "../svm_result/pred/%s_cvpred_%s.txt" % (data, mkl) pred = svm_mkl(rbf_km_list, y, mkl, 5, data,data) np.savetxt(pred_f, pred, fmt="%d")
class KernelPCA(BaseEstimator, TransformerMixin): """Kernel Principal component analysis (KPCA) Non-linear dimensionality reduction through the use of kernels (see :ref:`metrics`). Parameters ---------- n_components: int or None Number of components. If None, all non-zero components are kept. kernel: "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed" Kernel. Default: "linear" degree : int, default=3 Degree for poly kernels. Ignored by other kernels. gamma : float, optional Kernel coefficient for rbf and poly kernels. Default: 1/n_features. Ignored by other kernels. coef0 : float, optional Independent term in poly and sigmoid kernels. Ignored by other kernels. kernel_params : mapping of string to any, optional Parameters (keyword arguments) and values for kernel passed as callable object. Ignored by other kernels. alpha: int Hyperparameter of the ridge regression that learns the inverse transform (when fit_inverse_transform=True). Default: 1.0 fit_inverse_transform: bool Learn the inverse transform for non-precomputed kernels. (i.e. learn to find the pre-image of a point) Default: False eigen_solver: string ['auto'|'dense'|'arpack'] Select eigensolver to use. If n_components is much less than the number of training samples, arpack may be more efficient than the dense eigensolver. tol: float convergence tolerance for arpack. Default: 0 (optimal value will be chosen by arpack) max_iter : int maximum number of iterations for arpack Default: None (optimal value will be chosen by arpack) remove_zero_eig : boolean, default=True If True, then all components with zero eigenvalues are removed, so that the number of components in the output may be < n_components (and sometimes even zero due to numerical instability). When n_components is None, this parameter is ignored and components with zero eigenvalues are removed regardless. Attributes ---------- lambdas_ : Eigenvalues of the centered kernel matrix alphas_ : Eigenvectors of the centered kernel matrix evals_ : array[float], shape=(n_features) All eigenvalues of centered kernel matrix evecs_ : array[float, float], shape=(n_features, n_samples) All eigenvectors of centered kernel matrix dual_coef_ : Inverse transform matrix X_transformed_fit_ : Projection of the fitted data on the kernel principal components References ---------- Kernel PCA was introduced in: Bernhard Schoelkopf, Alexander J. Smola, and Klaus-Robert Mueller. 1999. Kernel principal component analysis. In Advances in kernel methods, MIT Press, Cambridge, MA, USA 327-352. """ def __init__(self, n_components=None, kernel="linear", gamma=None, degree=3, coef0=1, kernel_params=None, alpha=1.0, fit_inverse_transform=False, eigen_solver='auto', tol=0, max_iter=None, remove_zero_eig=False): if fit_inverse_transform and kernel == 'precomputed': raise ValueError( "Cannot fit_inverse_transform with a precomputed kernel.") self.n_components = n_components self.kernel = kernel self.kernel_params = kernel_params self.gamma = gamma self.degree = degree self.coef0 = coef0 self.alpha = alpha self.fit_inverse_transform = fit_inverse_transform self.eigen_solver = eigen_solver self.remove_zero_eig = remove_zero_eig self.tol = tol self.max_iter = max_iter self._centerer = KernelCenterer() @property def _pairwise(self): return self.kernel == "precomputed" def _get_kernel(self, X, Y=None): if callable(self.kernel): params = self.kernel_params or {} else: params = {"gamma": self.gamma, "degree": self.degree, "coef0": self.coef0} return pairwise_kernels(X, Y, metric=self.kernel, filter_params=True, **params) def _fit_transform(self, K): """ Fit's using kernel K""" # center kernel K = self._centerer.fit_transform(K) if self.n_components is None: n_components = K.shape[0] else: n_components = min(K.shape[0], self.n_components) # compute eigenvectors if self.eigen_solver == 'auto': if K.shape[0] > 200 and n_components < 10: eigen_solver = 'arpack' else: eigen_solver = 'dense' else: eigen_solver = self.eigen_solver if eigen_solver == 'dense': self.lambdas_, self.alphas_ = linalg.eigh( K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1)) self.evals_, self.evecs_ = linalg.eigh(K) elif eigen_solver == 'arpack': self.lambdas_, self.alphas_ = eigsh(K, n_components, which="LA", tol=self.tol, maxiter=self.max_iter) # sort eigenvectors in descending order indices = self.lambdas_.argsort()[::-1] self.lambdas_ = self.lambdas_[indices] self.alphas_ = self.alphas_[:, indices] # remove eigenvectors with a zero eigenvalue if self.remove_zero_eig or self.n_components is None: self.alphas_ = self.alphas_[:, self.lambdas_ > 0] self.lambdas_ = self.lambdas_[self.lambdas_ > 0] return K def _fit_inverse_transform(self, X_transformed, X): if hasattr(X, "tocsr"): raise NotImplementedError("Inverse transform not implemented for " "sparse matrices!") n_samples = X_transformed.shape[0] K = self._get_kernel(X_transformed) K.flat[::n_samples + 1] += self.alpha self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True) self.X_transformed_fit_ = X_transformed def fit(self, X, y=None): """Fit the model from data in X. Parameters ---------- X: array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the instance itself. """ K = self._get_kernel(X) self._fit_transform(K) if self.fit_inverse_transform: sqrt_lambdas = np.diag(np.sqrt(self.lambdas_)) X_transformed = np.dot(self.alphas_, sqrt_lambdas) self._fit_inverse_transform(X_transformed, X) self.X_fit_ = X return self def fit_transform(self, X, y=None, **params): """Fit the model from data in X and transform X. Parameters ---------- X: array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- X_new: array-like, shape (n_samples, n_components) """ self.fit(X, **params) X_transformed = self.alphas_ * np.sqrt(self.lambdas_) if self.fit_inverse_transform: self._fit_inverse_transform(X_transformed, X) return X_transformed def transform(self, X): """Transform X. Parameters ---------- X: array-like, shape (n_samples, n_features) Returns ------- X_new: array-like, shape (n_samples, n_components) """ check_is_fitted(self, 'X_fit_') K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_)) def inverse_transform(self, X): """Transform X back to original space. Parameters ---------- X: array-like, shape (n_samples, n_components) Returns ------- X_new: array-like, shape (n_samples, n_features) References ---------- "Learning to Find Pre-Images", G BakIr et al, 2004. """ if not self.fit_inverse_transform: raise NotFittedError("The fit_inverse_transform parameter was not" " set to True when instantiating and hence " "the inverse transform is not available.") K = self._get_kernel(X, self.X_transformed_fit_) return np.dot(K, self.dual_coef_)
#!/usr/bin/env python import numpy as np from sklearn.preprocessing import KernelCenterer from sklearn.metrics.pairwise import pairwise_kernels X = np.array([[ 1., -2., 2.], [ -2., 1., 3.], [ 4., 1., -2.]]) K = pairwise_kernels(X, metric='linear') transformer = KernelCenterer().fit(K) centered_K = transformer.transform(K) print(centered_K) H = np.eye(3) - (1.0/3)*np.ones((3,3)) centered_K = H.dot(K).dot(H) print(centered_K)
def ovkr_mkl(kernel_list, labels, mkl, n_folds, dataset, data): n_sample, n_labels = labels.shape n_km = len(kernel_list) tags = np.loadtxt("../data/cv/"+data+".cv") # Add noise to the output noise_level = [0.005, 0.010, 0.015, 0.020, 0.025] for nid in xrange(len(noise_level)): noi = noise_level[nid] print "noise", noi, nid Y = addNoise(labels, noi) pred = np.zeros((n_sample, n_labels)) pred_bin = np.zeros((n_sample, n_labels)) # Run for each fold for i in range(1,n_folds+1): print "Test fold %d" %i res_f = "../ovkr_result/noisy_weights/"+dataset+"_fold_%d_%s_noise_%d.weights" % (i,mkl, nid) # divide data test = np.array(tags == i) train = np.array(~test) train_y = Y[train,:] test_y = Y[test,:] n_train = len(train_y) n_test = len(test_y) train_km_list = [] test_km_list = [] for km in kernel_list: kc = KernelCenterer() train_km = km[np.ix_(train, train)] test_km = km[np.ix_(test, train)] # center train and test kernels kc.fit(train_km) train_km_c = kc.transform(train_km) test_km_c = kc.transform(test_km) train_km_list.append(train_km_c) test_km_list.append(test_km_c) if mkl == 'UNIMKL': wei = UNIMKL(n_km, n_labels) else: wei = np.loadtxt(res_f, ndmin=2) normw = np.linalg.norm(wei) uni = np.ones(n_km) / np.linalg.norm(np.ones(n_km)) if normw == 0: wei[:,0] = uni else: wei[:,0] = wei[:,0] / normw train_ckm = np.zeros((n_train,n_train)) for t in range(n_km): train_ckm += wei[t,0]*train_km_list[t] # combine train and test kernel using learned weights test_ckm = np.zeros(test_km_list[0].shape) for t in range(n_km): test_ckm = test_ckm + wei[t,0]*test_km_list[t] AP = OVKR_train_CV(train_ckm, train_y, tags[train]) pred_label = OVKR_test(test_ckm, AP) pred[test, :] = pred_label pred_real_f = "../ovkr_result/noisy_pred/%s_cvpred_%s_real_noise_%d.npy" % (data, mkl, nid) np.save(pred_real_f, pred)
# Kernel PLS if (FE_kPLS == 1): d = pair.pairwise_distances(Xtrain,Xtrain) aux = np.triu(d) sigma = np.sqrt(np.mean(np.power(aux[aux!=0],2)*0.5)) gamma = 1/(2*sigma**2) ktrain = pair.rbf_kernel(Xtrain,Xtrain,gamma) ktest = pair.rbf_kernel(Xtest,Xtrain,gamma) kcent = KernelCenterer() kcent.fit(ktrain) ktrain = kcent.transform(ktrain) ktest = kcent.transform(ktest) kpls = PLSRegression(n_components = n_comp) kpls.fit(ktrain,Ytrain_m) Xtrain = kpls.transform(ktrain) Xtest = kpls.transform(ktest) # Linear CCA Cannonical Correlation Análisis if (FE_CCA == 1): from sklearn.cross_decomposition import CCA cca = CCA(n_components = n_class) cca.fit(Xtrain,Ytrain_m) Xtrain = cca.transform(Xtrain)
def ALIGNFSOFT(kernel_list, ky, y, test_fold, tags): # Find best upper bound in CV and train on whole data # Reutrn the weights y = y.ravel() n_km = len(kernel_list) tag = np.array(tags) tag = tag[tag!=test_fold] remain_fold = np.unique(tag).tolist() all_best_c = [] for validate_fold in remain_fold: train = tag != validate_fold validate = tag == validate_fold # train on train fold ,validate on validate_fold. # Do not use test fold. test fold used in outter cv ky_train = ky[np.ix_(train, train)] y_train = y[train] y_validate = y[validate] train_km_list = [] validate_km_list = [] n_train = len(y_train) n_validate = len(y_validate) for km in kernel_list: kc = KernelCenterer() train_km = km[np.ix_(train, train)] validate_km = km[np.ix_(validate, train)] # center train and validate kernels train_km_c = kc.fit_transform(train_km) train_km_list.append(train_km_c) validate_km_c = kc.transform(validate_km) validate_km_list.append(validate_km_c) # if the label is too biased, SVM CV will fail, just return ALIGNF solution if np.sum(y_train==1) > n_train-3 or np.sum(y_train==-1) > n_train-3: return 1e8, ALIGNFSLACK(train_km_list, ky_train, 1e8) Cs = np.exp2(np.array(range(-9,7))).tolist() + [1e8] W = np.zeros((n_km, len(Cs))) for i in xrange(len(Cs)): W[:,i] = ALIGNFSLACK(train_km_list, ky_train, Cs[i]) W = W / np.linalg.norm(W, 2, 0) f1 = np.zeros(len(Cs)) for i in xrange(len(Cs)): train_ckm = np.zeros((n_train,n_train)) validate_ckm = np.zeros((n_validate,n_train)) w = W[:,i] for j in xrange(n_km): train_ckm += w[j]*train_km_list[j] validate_ckm += w[j]*validate_km_list[j] f1[i] = svm(train_ckm, validate_ckm, y_train, y_validate) # return the first maximum maxind = np.argmax(f1) bestC = Cs[maxind] all_best_c.append(bestC) print f1 print "..Best C is", bestC bestC = np.mean(all_best_c) print "..Take the average best upper bound", bestC # use the best upper bound to solve ALIGNFSOFT return bestC, ALIGNFSLACK(kernel_list, ky, bestC)
plt.xlim(1,np.amax(nComponents)) plt.title('kPCA accuracy') plt.xlabel('Number of components') plt.ylabel('accuracy') plt.xlim([500,1500]) plt.legend (['LDA'],loc='lower right') plt.grid(True) if(0): # K-PCA second round ktrain = pair.rbf_kernel(Xtrain,Xtrain,gamma) ktest = pair.rbf_kernel(Xtest,Xtrain,gamma) kcent = KernelCenterer() kcent.fit(ktrain) ktrain = kcent.transform(ktrain) ktest = kcent.transform(ktest) kpca = PCA() kpca.fit_transform(ktrain) cumvarkPCA2 = np.cumsum(kpca.explained_variance_ratio_[0:220]) # Calculate classifiation scores for each component nComponents = np.arange(1,nFeatures) kpcaScores2 = np.zeros((5,np.alen(nComponents))) for i,n in enumerate(nComponents): kpca2 = PCA(n_components=n) kpca2.fit(ktrain) XtrainT = kpca2.transform(ktrain) XtestT = kpca2.transform(ktest) kpcaScores2[:,i] = util.classify(XtrainT,XtestT,labelsTrain,labelsTest)
class KernelPCA(BaseEstimator, TransformerMixin): """Kernel Principal component analysis (KPCA) Non-linear dimensionality reduction through the use of kernels (see :ref:`metrics`). Parameters ---------- n_components: int or None Number of components. If None, all non-zero components are kept. kernel: "linear" | "poly" | "rbf" | "sigmoid" | "cosine" | "precomputed" Kernel. Default: "linear" degree : int, default=3 Degree for poly kernels. Ignored by other kernels. gamma : float, optional Kernel coefficient for rbf and poly kernels. Default: 1/n_features. Ignored by other kernels. coef0 : float, optional Independent term in poly and sigmoid kernels. Ignored by other kernels. kernel_params : mapping of string to any, optional Parameters (keyword arguments) and values for kernel passed as callable object. Ignored by other kernels. alpha: int Hyperparameter of the ridge regression that learns the inverse transform (when fit_inverse_transform=True). Default: 1.0 fit_inverse_transform: bool Learn the inverse transform for non-precomputed kernels. (i.e. learn to find the pre-image of a point) Default: False eigen_solver: string ['auto'|'dense'|'arpack'] Select eigensolver to use. If n_components is much less than the number of training samples, arpack may be more efficient than the dense eigensolver. tol: float convergence tolerance for arpack. Default: 0 (optimal value will be chosen by arpack) max_iter : int maximum number of iterations for arpack Default: None (optimal value will be chosen by arpack) remove_zero_eig : boolean, default=True If True, then all components with zero eigenvalues are removed, so that the number of components in the output may be < n_components (and sometimes even zero due to numerical instability). When n_components is None, this parameter is ignored and components with zero eigenvalues are removed regardless. Attributes ---------- lambdas_ : Eigenvalues of the centered kernel matrix alphas_ : Eigenvectors of the centered kernel matrix evals_ : array[float], shape=(n_features) All eigenvalues of centered kernel matrix evecs_ : array[float, float], shape=(n_features, n_samples) All eigenvectors of centered kernel matrix dual_coef_ : Inverse transform matrix X_transformed_fit_ : Projection of the fitted data on the kernel principal components References ---------- Kernel PCA was introduced in: Bernhard Schoelkopf, Alexander J. Smola, and Klaus-Robert Mueller. 1999. Kernel principal component analysis. In Advances in kernel methods, MIT Press, Cambridge, MA, USA 327-352. """ def __init__(self, n_components=None, kernel="linear", gamma=None, degree=3, coef0=1, kernel_params=None, alpha=1.0, fit_inverse_transform=False, eigen_solver='auto', tol=0, max_iter=None, remove_zero_eig=False): if fit_inverse_transform and kernel == 'precomputed': raise ValueError( "Cannot fit_inverse_transform with a precomputed kernel.") self.n_components = n_components self.kernel = kernel self.kernel_params = kernel_params self.gamma = gamma self.degree = degree self.coef0 = coef0 self.alpha = alpha self.fit_inverse_transform = fit_inverse_transform self.eigen_solver = eigen_solver self.remove_zero_eig = remove_zero_eig self.tol = tol self.max_iter = max_iter self._centerer = KernelCenterer() @property def _pairwise(self): return self.kernel == "precomputed" def _get_kernel(self, X, Y=None): if callable(self.kernel): params = self.kernel_params or {} else: params = { "gamma": self.gamma, "degree": self.degree, "coef0": self.coef0 } return pairwise_kernels(X, Y, metric=self.kernel, filter_params=True, **params) def _fit_transform(self, K): """ Fit's using kernel K""" # center kernel K = self._centerer.fit_transform(K) if self.n_components is None: n_components = K.shape[0] else: n_components = min(K.shape[0], self.n_components) # compute eigenvectors if self.eigen_solver == 'auto': if K.shape[0] > 200 and n_components < 10: eigen_solver = 'arpack' else: eigen_solver = 'dense' else: eigen_solver = self.eigen_solver if eigen_solver == 'dense': self.lambdas_, self.alphas_ = linalg.eigh( K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1)) self.evals_, self.evecs_ = linalg.eigh(K) elif eigen_solver == 'arpack': self.lambdas_, self.alphas_ = eigsh(K, n_components, which="LA", tol=self.tol, maxiter=self.max_iter) # sort eigenvectors in descending order indices = self.lambdas_.argsort()[::-1] self.lambdas_ = self.lambdas_[indices] self.alphas_ = self.alphas_[:, indices] # remove eigenvectors with a zero eigenvalue if self.remove_zero_eig or self.n_components is None: self.alphas_ = self.alphas_[:, self.lambdas_ > 0] self.lambdas_ = self.lambdas_[self.lambdas_ > 0] return K def _fit_inverse_transform(self, X_transformed, X): if hasattr(X, "tocsr"): raise NotImplementedError("Inverse transform not implemented for " "sparse matrices!") n_samples = X_transformed.shape[0] K = self._get_kernel(X_transformed) K.flat[::n_samples + 1] += self.alpha self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True) self.X_transformed_fit_ = X_transformed def fit(self, X, y=None): """Fit the model from data in X. Parameters ---------- X: array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the instance itself. """ K = self._get_kernel(X) self._fit_transform(K) if self.fit_inverse_transform: sqrt_lambdas = np.diag(np.sqrt(self.lambdas_)) X_transformed = np.dot(self.alphas_, sqrt_lambdas) self._fit_inverse_transform(X_transformed, X) self.X_fit_ = X return self def fit_transform(self, X, y=None, **params): """Fit the model from data in X and transform X. Parameters ---------- X: array-like, shape (n_samples, n_features) Training vector, where n_samples in the number of samples and n_features is the number of features. Returns ------- X_new: array-like, shape (n_samples, n_components) """ self.fit(X, **params) X_transformed = self.alphas_ * np.sqrt(self.lambdas_) if self.fit_inverse_transform: self._fit_inverse_transform(X_transformed, X) return X_transformed def transform(self, X): """Transform X. Parameters ---------- X: array-like, shape (n_samples, n_features) Returns ------- X_new: array-like, shape (n_samples, n_components) """ check_is_fitted(self, 'X_fit_') K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) return np.dot(K, self.alphas_ / np.sqrt(self.lambdas_)) def inverse_transform(self, X): """Transform X back to original space. Parameters ---------- X: array-like, shape (n_samples, n_components) Returns ------- X_new: array-like, shape (n_samples, n_features) References ---------- "Learning to Find Pre-Images", G BakIr et al, 2004. """ if not self.fit_inverse_transform: raise NotFittedError("The fit_inverse_transform parameter was not" " set to True when instantiating and hence " "the inverse transform is not available.") K = self._get_kernel(X, self.X_transformed_fit_) return np.dot(K, self.dual_coef_)
class KernelPCA(TransformerMixin, BaseEstimator): def __init__(self, kernel="linear", gamma=None, degree=3, coef0=1, kernel_params=None, alpha=1.0, fit_inverse_transform=False, eigen_solver='auto', tol=0, max_iter=None, remove_zero_eig=False, random_state=None, copy_X=True, n_jobs=None): if fit_inverse_transform and kernel == 'precomputed': raise ValueError( "Cannot fit_inverse_transform with a precomputed kernel.") self.kernel = kernel self.kernel_params = kernel_params self.gamma = gamma self.degree = degree self.coef0 = coef0 self.alpha = alpha self.fit_inverse_transform = fit_inverse_transform self.eigen_solver = eigen_solver self.remove_zero_eig = remove_zero_eig self.tol = tol self.max_iter = max_iter self.random_state = random_state self.n_jobs = n_jobs self.copy_X = copy_X @property def _pairwise(self): return self.kernel == "precomputed" def _get_kernel(self, X, Y=None): if callable(self.kernel): params = self.kernel_params or {} else: params = { "gamma": self.gamma, "degree": self.degree, "coef0": self.coef0 } return pairwise_kernels(X, Y, metric=self.kernel, filter_params=True, n_jobs=self.n_jobs, **params) def _fit_transform(self, K): """ Fit's using kernel K""" # center kernel K = self._centerer.fit_transform(K) return K def _fit_inverse_transform(self, X_transformed, X): if hasattr(X, "tocsr"): raise NotImplementedError("Inverse transform not implemented for " "sparse matrices!") n_samples = X_transformed.shape[0] K = self._get_kernel(X_transformed) K.flat[::n_samples + 1] += self.alpha self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True) self.X_transformed_fit_ = X_transformed def fit(self, X, y=None): X = check_array(X, accept_sparse='csr', copy=self.copy_X) self._centerer = KernelCenterer() K = self._get_kernel(X) self._fit_transform(K) if self.fit_inverse_transform: # no need to use the kernel to transform X, use shortcut expression X_transformed = self.alphas_ * np.sqrt(self.lambdas_) self._fit_inverse_transform(X_transformed, X) self.X_fit_ = X return self def fit_transform(self, X, y=None, **params): self.fit(X, **params) # no need to use the kernel to transform X, use shortcut expression X_transformed = self.alphas_ * np.sqrt(self.lambdas_) if self.fit_inverse_transform: self._fit_inverse_transform(X_transformed, X) return X_transformed def transform(self, X): check_is_fitted(self) # Compute centered gram matrix between X and training data X_fit_ K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) # scale eigenvectors (properly account for null-space for dot product) non_zeros = np.flatnonzero(self.lambdas_) scaled_alphas = np.zeros_like(self.alphas_) scaled_alphas[:, non_zeros] = (self.alphas_[:, non_zeros] / np.sqrt(self.lambdas_[non_zeros])) # Project with a scalar product between K and the scaled eigenvectors return np.dot(K, scaled_alphas) def inverse_transform(self, X): if not self.fit_inverse_transform: raise NotFittedError("The fit_inverse_transform parameter was not" " set to True when instantiating and hence " "the inverse transform is not available.") K = self._get_kernel(X, self.X_transformed_fit_) return np.dot(K, self.dual_coef_)