def test_nystroem_callable(): # Test Nystroem on a callable. rnd = np.random.RandomState(42) n_samples = 10 X = rnd.uniform(size=(n_samples, 4)) def logging_histogram_kernel(x, y, log): """Histogram kernel that writes to a log.""" log.append(1) return np.minimum(x, y).sum() kernel_log = [] X = list(X) # test input validation Nystroem(kernel=logging_histogram_kernel, n_components=(n_samples - 1), kernel_params={'log': kernel_log}).fit(X) assert len(kernel_log) == n_samples * (n_samples - 1) / 2 # if degree, gamma or coef0 is passed, we raise a warning msg = "Don't pass gamma, coef0 or degree to Nystroem" params = ({'gamma': 1}, {'coef0': 1}, {'degree': 2}) for param in params: ny = Nystroem(kernel=_linear_kernel, **param) with pytest.raises(ValueError, match=msg): ny.fit(X)
def test_nystroem_approximation(): # some basic tests rnd = np.random.RandomState(0) X = rnd.uniform(size=(10, 4)) # With n_components = n_samples this is exact X_transformed = Nystroem(n_components=X.shape[0]).fit_transform(X) K = rbf_kernel(X) assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K) trans = Nystroem(n_components=2, random_state=rnd) X_transformed = trans.fit(X).transform(X) assert_equal(X_transformed.shape, (X.shape[0], 2)) # test callable kernel linear_kernel = lambda X, Y: np.dot(X, Y.T) trans = Nystroem(n_components=2, kernel=linear_kernel, random_state=rnd) X_transformed = trans.fit(X).transform(X) assert_equal(X_transformed.shape, (X.shape[0], 2)) # test that available kernels fit and transform kernels_available = kernel_metrics() for kern in kernels_available: trans = Nystroem(n_components=2, kernel=kern, random_state=rnd) X_transformed = trans.fit(X).transform(X) assert_equal(X_transformed.shape, (X.shape[0], 2))
class NystroemImpl(): def __init__(self, kernel='rbf', gamma=None, coef0=None, degree=None, kernel_params=None, n_components=100, random_state=None): self._hyperparams = { 'kernel': kernel, 'gamma': gamma, 'coef0': coef0, 'degree': degree, 'kernel_params': kernel_params, 'n_components': n_components, 'random_state': random_state } self._wrapped_model = Op(**self._hyperparams) def fit(self, X, y=None): if (y is not None): self._wrapped_model.fit(X, y) else: self._wrapped_model.fit(X) return self def transform(self, X): return self._wrapped_model.transform(X)
def test_nystroem_callable(): # Test Nystroem on a callable. rnd = np.random.RandomState(42) n_samples = 10 X = rnd.uniform(size=(n_samples, 4)) def logging_histogram_kernel(x, y, log): """Histogram kernel that writes to a log.""" log.append(1) return np.minimum(x, y).sum() kernel_log = [] X = list(X) # test input validation Nystroem(kernel=logging_histogram_kernel, n_components=(n_samples - 1), kernel_params={'log': kernel_log}).fit(X) assert_equal(len(kernel_log), n_samples * (n_samples - 1) / 2) def linear_kernel(X, Y): return np.dot(X, Y.T) # if degree, gamma or coef0 is passed, we raise a warning msg = "Don't pass gamma, coef0 or degree to Nystroem" params = ({'gamma': 1}, {'coef0': 1}, {'degree': 2}) for param in params: ny = Nystroem(kernel=linear_kernel, **param) with pytest.raises(ValueError, match=msg): ny.fit(X)
class Kernel_tica(object): def __init__( self, n_components, lag_time, gamma, # gamma value for rbf kernel n_components_nystroem=100, # number of components for Nystroem kernel approximation landmarks=None, shrinkage=None, weights='empirical' # if 'koopman', use Koopman reweighting for tICA (see Wu, Hao, et al. "Variational Koopman models: slow collective variables and molecular kinetics from short off-equilibrium simulations." The Journal of Chemical Physics 146.15 (2017): 154104.) ): self._n_components = n_components self._lag_time = lag_time self._n_components_nystroem = n_components_nystroem self._landmarks = landmarks self._gamma = gamma self._nystroem = Nystroem(gamma=gamma, n_components=n_components_nystroem) self._weights = weights # self._tica = tICA(n_components=n_components, lag_time=lag_time, shrinkage=shrinkage) self._shrinkage = shrinkage return def fit(self, sequence_list): if self._landmarks is None: self._nystroem.fit(np.concatenate(sequence_list)) else: print("using landmarks") self._nystroem.fit(self._landmarks) sequence_transformed = [ self._nystroem.transform(item) for item in sequence_list ] # define tica object at fit() with sequence_list supplied for initialization, as it is required by # Koopman reweighting self._tica = py.coordinates.tica(sequence_transformed, lag=self._lag_time, dim=self._n_components, kinetic_map=True, weights=self._weights) return def transform(self, sequence_list): return self._tica.transform( [self._nystroem.transform(item) for item in sequence_list]) def fit_transform(self, sequence_list): self.fit(sequence_list) return self.transform(sequence_list) def score(self, sequence_list): model = self.__class__( n_components=self._n_components, lag_time=self._lag_time, gamma=self._gamma, n_components_nystroem=self._n_components_nystroem, landmarks=self._landmarks, shrinkage=self._shrinkage) model.fit(sequence_list) return np.sum(model._tica.eigenvalues)
def init_scaler(self, observed_cmd, gamma=0.5): self.cmd_scaler = MinMaxScaler() self.cmd_scaler.fit(observed_cmd) scaled_observed_cmd = self.cmd_scaler.transform(observed_cmd) Phi_approx = Nystroem(kernel='rbf', n_components=50, gamma=gamma) Phi_approx.fit(scaled_observed_cmd) self.mapping = Phi_approx.transform print('scaler initialized and mapping defined!') return scaled_observed_cmd
def test_nystroem_component_indices(): """Check that `component_indices_` corresponds to the subset of training points used to construct the feature map. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/20474 """ X, _ = make_classification(n_samples=100, n_features=20) feature_map_nystroem = Nystroem( n_components=10, random_state=0, ) feature_map_nystroem.fit(X) assert feature_map_nystroem.component_indices_.shape == (10, )
class NystroemImpl: def __init__(self, **hyperparams): self._hyperparams = hyperparams self._wrapped_model = Op(**self._hyperparams) def fit(self, X, y=None): if y is not None: self._wrapped_model.fit(X, y) else: self._wrapped_model.fit(X) return self def transform(self, X): return self._wrapped_model.transform(X)
def test_nystroem_precomputed_kernel(): # Non-regression: test Nystroem on precomputed kernel. # PR - 14706 rnd = np.random.RandomState(12) X = rnd.uniform(size=(10, 4)) K = polynomial_kernel(X, degree=2, coef0=0.1) nystroem = Nystroem(kernel="precomputed", n_components=X.shape[0]) X_transformed = nystroem.fit_transform(K) assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K) # if degree, gamma or coef0 is passed, we raise a ValueError msg = "Don't pass gamma, coef0 or degree to Nystroem" params = ({"gamma": 1}, {"coef0": 1}, {"degree": 2}) for param in params: ny = Nystroem(kernel="precomputed", n_components=X.shape[0], **param) with pytest.raises(ValueError, match=msg): ny.fit(K)
def kernel_transformation_using_nystroem_rbf(self,df,cat): df=df.fillna(0) df=df.replace([np.inf, -np.inf], 0) datecol=[x for x in df.columns if df[x].dtypes=='datetime64[ns]'] X1=[x for x in df.columns if df[x].dtypes != 'object' and x not in datecol and x not in self.target] X=[x for x in X1 if x not in cat] y=self.target j = np.linspace((10**-2),(10**2),50) g=0 max1=0 df=df.fillna(0) df=df.replace(np.inf, 0) df=df.replace(-np.inf, 0) for i in j: rbf_feature = Nystroem(kernel = 'rbf', gamma=i, random_state=2,n_components=10) rbf_feature.fit(df[X]) X_features = rbf_feature.transform(df[X]) X_features=np.nan_to_num(X_features) # SGDClassifier(loss='hinge', penalty='l2', alpha=0.0001, l1_ratio=0.15, fit_intercept=True, shuffle=True, verbose=0, epsilon=0.1, n_jobs=1, random_state=None, learning_rate='optimal', eta0=0.0, power_t=0.5, class_weight=None, warm_start=False, average=False, n_iter=None) clf = SGDClassifier() clf.fit(X_features, df[y]) y_pred = clf.predict(X_features) score=f1_score(df[y], y_pred, average='micro') if(score>max1): max1=score g=i rbf_feature = RBFSampler(gamma=g, random_state=2,n_components=10) rbf_feature.fit(df[X]) X_features = rbf_feature.transform(df[X]) l=[] for r in range(10): l.append('k_'+str(r)) X_features=pd.DataFrame(data=X_features,columns=l) # SGDClassifier(loss='hinge', penalty='l2', alpha=0.0001, l1_ratio=0.15, fit_intercept=True,shuffle=True, verbose=0, epsilon=0.1, n_jobs=1, random_state=None, learning_rate='optimal', eta0=0.0, power_t=0.5, class_weight=None, warm_start=False, average=False, n_iter=None) clf = SGDClassifier() clf.fit(X_features, df[y]) score=f1_score(df[y], y_pred, average='micro') print("Score is") print(score) print(g) return X_features
class NystromScikit: """ Nystrom implementation form Scikit Learn wrapper. The main difference is in selection of inducing inputs. """ def __init__(self, rank=10, random_state=42): """ :param rank: (``int``) Maximal decomposition rank. :param random_state: (``int``) Random generator seed. """ self.trained = False self.rank = rank self.random_state = random_state def fit(self, K, y): """ Fit approximation to the kernel function / matrix. :param K: (``numpy.ndarray``) or of (``Kinterface``). The kernel to be approximated with G. :param y: (``numpy.ndarray``) Class labels :math:`y_i \in {-1, 1}` or regression targets. """ assert isinstance(K, Kinterface) self.n = K.shape[0] kernel = lambda x, y: K.kernel(x, y, **K.kernel_args) self.model = Nystroem(kernel=kernel, n_components=self.rank, random_state=self.random_state) self.model.fit(K.data, y) self.active_set_ = list(self.model.component_indices_[:self.rank]) assert len(set(self.active_set_)) == len(self.active_set_) == self.rank R = self.model.normalization_ self.G = K[:, self.active_set_].dot(R) self.trained = True
def nystroem_pca(X, nystroem_kwargs, pca_kwargs, verbose=1): """ Perform Nystroem kernel approximation and then PCA decomposition. The Nystroem method constructs an approximate feature map, for an arbitrary kernel using a subset of the data as basis. We then apply this feature map to X, and perform PCA decomposition in the new feature space. :param X: Values of our data :param nystroem_kwargs: (dict) Keyword arguments that are passed to the constructor of an implementation of the Nystroem method - `sklearn.kernel_approximation.Nystroem`. :param pca_kwargs: dict) Keyword arguments that are passed to `sklearn.decomposition.PCA`. :param verbose: (int) If verbose=1, print information relevant to the PCA decomposition. Set verbose=0 for no output. :return: `numpy.ndarray`: The PCA decomposition of our data set in the new feature space, that was approximated via the nystroem method. """ # Get the nystroem approximation nystroem = Nystroem(**nystroem_kwargs) nystroem.fit(X) X_nystroem = nystroem.transform(X) # Get the PCA decomposition pca_nystroem = PCA(**pca_kwargs) pca_nystroem.fit(X_nystroem) X_nystroem_pca = pca_nystroem.transform(X_nystroem) # Print Relevant PCA information if verbose: print("Dimensionality reduction: {} -> {}".format( X_nystroem.shape[1], X_nystroem_pca.shape[1])) print("Variance explained by each component:", (pca_nystroem.explained_variance_ratio_ * 1000).astype(int) / 1000) print( "Total variance explained by those {} components:".format( X_nystroem_pca.shape[1]), format(pca_nystroem.explained_variance_ratio_.sum(), ".4f")) return X_nystroem_pca
def main(): data = pd.DataFrame(circle(3000), columns=['x', 'y']) ns = Nystroem(n_components=3000, gamma=0.15) ns.fit(data) fullsample = pd.DataFrame( [(x, y) for x in np.around( np.linspace(-20.0, 20.0, 128), decimals=2, ) for y in np.around( np.linspace(-20.0, 20.0, 128), decimals=2, )], columns=['x', 'y'], ) transformed = ns.transform(fullsample) fullsample['c'] = pd.Series(transformed[:, 0]) sns.heatmap(fullsample.pivot('x', 'y', 'c'), xticklabels=32, yticklabels=32) pyplot.show()
def construct_Q(self, feature_normalization=True): """ Step 1 and 2 in Algorithm 1 in MSc. Thesis. Computes Q, such that QQ^T \approx K, where K is the RBF kernel matrix. Also applies normalization to the features by default. This is carried over by example from the CCNN code of Zhang et. al Input Z_train, Z_test: (N,P,d1) arrays. Each Z[i,:,:] is one Z(x_i). Result from __init__. Output Q_train, Q_test: (N,P,m) arrays Each Q[i,:,:] is one Q(x_i). Used in train() function below. """ from sklearn.kernel_approximation import Nystroem import numpy as np import math tprint("Using Scikitlearn Nystroem function") tprint("Creating Q...") Z_train = self.Z[0:self.n_train].reshape( (self.n_train * self.P, self.d1)) Z_test = self.Z[self.n_train:self.n].reshape( (self.n_test * self.P, self.d1)) transformer = Nystroem(gamma=self.gamma, n_components=self.nystrom_dim) transformer = transformer.fit(X=Z_train) Q_train = transformer.transform(Z_train) Q_test = transformer.transform(Z_test) self.Q_train = Q_train.reshape( (self.n_train, self.P, self.nystrom_dim)) self.Q_test = Q_test.reshape((self.n_test, self.P, self.nystrom_dim)) if feature_normalization == True: self.Q_train = self.Q_train.reshape( (self.n_train * self.P, self.nystrom_dim)) self.Q_train -= np.mean(self.Q_train, axis=0) self.Q_train /= LA.norm(self.Q_train) / math.sqrt( self.n_train * self.P) self.Q_train = self.Q_train.reshape( (self.n_train, self.P, self.nystrom_dim)) self.Q_test = self.Q_test.reshape( (self.n_test * self.P, self.nystrom_dim)) self.Q_test -= np.mean(self.Q_test, axis=0) self.Q_test /= LA.norm(self.Q_test) / math.sqrt( self.n_train * self.P) self.Q_test = self.Q_test.reshape( (self.n_test, self.P, self.nystrom_dim))
from params import ts_depths,n_fea,gamma, np, sp class Whitener: def __init__(self,X): self.Xmean = X.mean(0) self.Xstd = X.std(0) def whiten(self,Z): return (Z-self.Xmean)/self.Xstd def unwhiten(self,Zw): return Zw*self.Xstd + self.Xmean def expkern(x,y): return np.exp(-gamma*la.norm(x-y)) wh = Whitener(ts_depths) ts_depths_w = wh.whiten(ts_depths) xx = np.linspace(ts_depths_w.min(),ts_depths_w.max(),n_fea)[:,np.newaxis] rbf_tr = Nystroem(expkern,gamma,n_components=n_fea) #rbf_tr = Nystroem(gamma=gamma,n_components=n_fea) #class rbf_transformer: # def __init__(self,X,gamma): # self.X = X # self.gamma = gamma # def transform(self,xx): # return rbf_kernel(xx,self.X) #rbf_tr = rbf_transformer(xx,gamma) #rbf_tr.fit(x) rbf_tr.fit(xx) ts_depths_tr = rbf_tr.transform(ts_depths_w)
dataTrainT = kcca.transform(ktrain) dataTestT = kcca.transform(ktest) kccaScores = np.zeros((2,np.alen(nComponents))) for i,n in enumerate(nComponents): kccaScores[:,i] = util.classify(dataTrainT[:,0:n],dataTestT[:,0:n],labelsTrain,labelsTest) #%% Subsampling methods kpls = PLSRegression(n_components=150) nComponents = np.arange(173,2173,100) # Nystroem method elapTimeNys = np.zeros(np.shape(nComponents)) kplsScoresNys = np.zeros((2,3)) for i,n in enumerate(nComponents): nys = Nystroem(n_components=n,gamma=gamma) nys.fit(dataTrain) ktrain = nys.transform(dataTrain) ktest = nys.transform(dataTest) startTime = timeit.default_timer() kpls.fit(ktrain,Ytrain) elapTimeNys[i] = timeit.default_timer() - startTime dataTrainT = kpls.transform(ktrain) dataTestT = kpls.transform(ktest) if n==573: kplsScoresNys[:,0] = util.classify(dataTrainT,dataTestT,labelsTrain,labelsTest) elif n==1073: kplsScoresNys[:,1] = util.classify(dataTrainT,dataTestT,labelsTrain,labelsTest) elif n==1573: kplsScoresNys[:,2] = util.classify(dataTrainT,dataTestT,labelsTrain,labelsTest)
class ParametricModelApproximation(object): """Approximate a Gaussian Process by a parametric model. Approximating a Gaussian Process by a parametric model can be useful if one has to evaluate a sample function from the GP repeatedly or on many evaluation points as this would become computationally very expensive with a GP. Parameters ---------- model : GaussianProcessRegressor The Gaussian Process which is to be approximated bounds: list of pair of floats The boundaries of the data space. This is used when determining the features of the parametric approximation (they are centered at random points in the data space) n_components: int The number of features/parameters of the parametric model seed: int The seed of the random number generator """ def __init__(self, model, bounds, n_components, seed): self.gp = model self.bounds = bounds self.n_components = n_components self.rng = np.random.RandomState(seed) self.X_space = self.rng.uniform(self.bounds[:, 0], self.bounds[:, 1], (1000, self.bounds.shape[0])) assert self.gp.X_fit_.shape[1] == self.X_space.shape[1] self.kernel = self.gp.kernel_ self.nystr = Nystroem( n_components=min(self.n_components, self.X_space.shape[0]), kernel='precomputed', random_state=self.rng) self.nystr.fit(self.kernel(self.X_space)) def determine_coefs(self, X_query=None, y_query_samples=None, n_samples=1): """ Determine coefficients of parametric model. Simulate an evaluation at X_query with outcomes y_query_samples. Determine coefficients of parametric model the updated GP. Parameters ---------- X_query : ndarray-like, default: None The query point at which an additional evaluation is simulated. If None, a parametric approximation of the unmodified GP is returned. y_query_samples: ndarray-like, default: None The possible outcomes of a query at X_query. n_samples: int The number of independent samples of model coefficients from the Bayesian posterior over model coefficients """ if X_query is not None: X_query = np.asarray(X_query) X_queried = np.vstack((self.gp.X_fit_, X_query)) else: X_queried = self.gp.X_fit_ y_queried = self.gp.y_fit_ Phi = self.nystr.transform(self.kernel(self.X_space, X_queried)) A = Phi.T.dot(Phi) + self.gp.alpha * np.eye(Phi.shape[1]) A_inv = np.linalg.inv(A) cov = self.gp.alpha * A_inv coefs = \ np.empty((n_samples, self.n_components, y_query_samples.shape[0])) for i in range(y_query_samples.shape[0]): # XXX: Vectorize y_queried = np.hstack((self.gp.y_fit_, y_query_samples[i])) mean = A_inv.dot(Phi.T).dot(y_queried) coefs[:, :, i] = self.rng.multivariate_normal(mean, cov, n_samples) return np.array(coefs) def __call__(self, X, coefs): """ Evaluate parametric model at X for the given sampled coefficients. Parameters ---------- X : ndarray-like The points at which the parametric model is to be evaluated coefs: ndarray-like The coefficients of the parametric model. """ X = np.atleast_2d(X) Phi = self.nystr.transform(self.kernel(self.X_space, X)) f = Phi.dot(coefs) return f
width=1))) fig.append_trace(projection_fourier_1, 1, 2) fig.append_trace(projection_fourier_2, 1, 2) fig['layout']['xaxis2'].update(title='1st principal component', zeroline=False, showgrid=False) fig['layout']['yaxis2'].update(title='2nd component', zeroline=False, showgrid=False) ## Nystroem nystroem = Nystroem(gamma=gamma, random_state=1) nystroem.fit(X_pca) X_nystroem_pca = nystroem.transform(X_pca) projection_nystroem_1 = go.Scatter(x=X_nystroem_pca[reds, 0], y=X_nystroem_pca[reds, 1], mode='markers', showlegend=False, marker=dict(color='red', line=dict(color='black', width=1))) projection_nystroem_2 = go.Scatter(x=X_nystroem_pca[blues, 0], y=X_nystroem_pca[blues, 1], mode='markers', showlegend=False, marker=dict(color='blue', line=dict(color='black',
def make_nystrom_evaluation(x_train, y_train, x_test, y_test, U_centroids): """ Evaluation Nystrom construction time and approximation precision. The approximation is based on a subsample of size n_sample of the input data set. :param x_train: Input dataset as ndarray. :param U_centroids: The matrix of centroids as ndarray or SparseFactor object :param n_sample: The number of sample to take into account in the reconstruction (can't be too large) :return: """ n_sample = paraman["--nystrom"] if n_sample > x_train.shape[0]: logger.warning( "Batch size for nystrom evaluation is bigger than data size. {} > {}. Using " "data size instead.".format(n_sample, x_train.shape[0])) n_sample = x_train.shape[0] paraman["--nystrom"] = n_sample # Compute euristic gamma as the mean of euclidian distance between example gamma = compute_euristic_gamma(x_train) log_memory_usage( "Memory after euristic gamma computation in make_nystrom_evaluation") # precompute the centroids norm for later use (optimization) centroids_norm = get_squared_froebenius_norm_line_wise(U_centroids) # centroids_norm = None indexes_samples = np.random.permutation(x_train.shape[0])[:n_sample] sample = x_train[indexes_samples] samples_norm = None log_memory_usage( "Memory after sample selection in make_nystrom_evaluation") ######################## # Nystrom on centroids # ######################## logger.info("Build Nystrom on centroids") ## TIME: nystrom build time # nystrom build time is Nystrom preparation time for later use. ## START nystrom_build_start_time = time.process_time() metric = prepare_nystrom(U_centroids, centroids_norm, gamma=gamma) nystrom_build_stop_time = time.process_time() log_memory_usage("Memory after SVD computation in make_nystrom_evaluation") # STOP nystrom_build_time = nystrom_build_stop_time - nystrom_build_start_time ## TIME: nystrom inference time # Nystrom inference time is the time for Nystrom transformation for all the samples. ## START nystrom_inference_time_start = time.process_time() nystrom_embedding = nystrom_transformation(sample, U_centroids, metric, centroids_norm, samples_norm, gamma=gamma) nystrom_approx_kernel_value = nystrom_embedding @ nystrom_embedding.T nystrom_inference_time_stop = time.process_time() log_memory_usage( "Memory after kernel matrix approximation in make_nystrom_evaluation") ## STOP nystrom_inference_time = (nystrom_inference_time_stop - nystrom_inference_time_start) / n_sample ################################################################ ###################### # Nystrom on uniform # ###################### logger.info("Build Nystrom on uniform sampling") indexes_uniform_samples = np.random.permutation( x_train.shape[0])[:U_centroids.shape[0]] uniform_sample = x_train[indexes_uniform_samples] uniform_sample_norm = None log_memory_usage( "Memory after uniform sample selection in make_nystrom_evaluation") metric_uniform = prepare_nystrom(uniform_sample, uniform_sample_norm, gamma=gamma) log_memory_usage( "Memory after SVD computation in uniform part of make_nystrom_evaluation" ) nystrom_embedding_uniform = nystrom_transformation(sample, uniform_sample, metric_uniform, uniform_sample_norm, samples_norm, gamma=gamma) nystrom_approx_kernel_value_uniform = nystrom_embedding_uniform @ nystrom_embedding_uniform.T ################################################################# ############### # Real Kernel # ############### logger.info("Compute real kernel matrix") real_kernel_special = special_rbf_kernel(sample, sample, gamma, norm_X=samples_norm, norm_Y=samples_norm) real_kernel = rbf_kernel(sample, sample, gamma) real_kernel_norm = np.linalg.norm(real_kernel) log_memory_usage( "Memory after real kernel computation in make_nystrom_evaluation") ################################# # Sklearn based Nystrom uniform # ################################# sklearn_nystrom = Nystroem(gamma=gamma, n_components=uniform_sample.shape[0]) sklearn_nystrom = sklearn_nystrom.fit(uniform_sample) sklearn_transfo = sklearn_nystrom.transform(sample) kernel_sklearn_nys = sklearn_transfo @ sklearn_transfo.T ################################################################ #################### # Error evaluation # #################### sampled_froebenius_norm = np.linalg.norm(nystrom_approx_kernel_value - real_kernel) / real_kernel_norm sampled_froebenius_norm_uniform = np.linalg.norm( nystrom_approx_kernel_value_uniform - real_kernel) / real_kernel_norm # svm evaluation if x_test is not None: logger.info("Start classification") time_classification_start = time.process_time() x_train_nystrom_embedding = nystrom_transformation(x_train, U_centroids, metric, centroids_norm, None, gamma=gamma) x_test_nystrom_embedding = nystrom_transformation(x_test, U_centroids, metric, centroids_norm, None, gamma=gamma) linear_svc_clf = LinearSVC() linear_svc_clf.fit(x_train_nystrom_embedding, y_train) accuracy_nystrom_svm = linear_svc_clf.score(x_test_nystrom_embedding, y_test) time_classification_stop = time.process_time() delta_time_classification = time_classification_stop - time_classification_start else: accuracy_nystrom_svm = None delta_time_classification = None nystrom_results = { "nystrom_build_time": nystrom_build_time, "nystrom_inference_time": nystrom_inference_time, "nystrom_sampled_error_reconstruction": sampled_froebenius_norm, "nystrom_sampled_error_reconstruction_uniform": sampled_froebenius_norm_uniform, "nystrom_svm_accuracy": accuracy_nystrom_svm, "nystrom_svm_time": delta_time_classification } resprinter.add(nystrom_results)
def kernel_approximation(self, x, param_info): #kapp = RBFSampler(gamma=param_info.kapp_gamma, param_info.kapp_num) kapp = Nystroem(gamma=param_info.kapp_gamma, n_components=param_info.kapp_num) self.learned_kapp = kapp.fit(x)
class KernelLogisticRegression(LogisticRegression): """A simple kernel logistic implementation using a Nystroem kernel approximation Warnings -------- This kernel method is not specialized for temporal classification. See Also -------- wildboar.datasets.outlier.EmmottLabeler : Synthetic outlier dataset construction """ def __init__(self, kernel=None, *, kernel_params=None, n_components=100, penalty="l2", dual=False, tol=1e-4, C=1.0, fit_intercept=True, intercept_scaling=1, class_weight=None, random_state=None, solver="lbfgs", max_iter=100, multi_class="auto", verbose=0, warm_start=False, n_jobs=None, l1_ratio=None): """Create a new kernel logistic regression Parameters ---------- kernel : str, optional The kernel function to use. See `sklearn.metrics.pairwise.kernel_metric` for kernels. The default kernel is 'rbf'. kernel_params : dict, optional Parameters to the kernel function. n_components : int, optional Number of features to construct """ super().__init__( penalty=penalty, dual=dual, tol=tol, C=C, fit_intercept=fit_intercept, intercept_scaling=intercept_scaling, class_weight=class_weight, random_state=random_state, solver=solver, max_iter=max_iter, multi_class=multi_class, verbose=verbose, warm_start=warm_start, n_jobs=n_jobs, l1_ratio=l1_ratio, ) self.kernel = kernel self.kernel_params = kernel_params self.n_components = n_components def fit(self, x, y, sample_weight=None): random_state = check_random_state(self.random_state) kernel = self.kernel or "rbf" n_components = min(x.shape[0], self.n_components) self.nystroem_ = Nystroem( kernel=kernel, kernel_params=self.kernel_params, n_components=n_components, random_state=random_state.randint(np.iinfo(np.int32).max), ) self.nystroem_.fit(x) super().fit(self.nystroem_.transform(x), y, sample_weight=sample_weight) return self def decision_function(self, x): check_is_fitted(self) return super().decision_function(self.nystroem_.transform(x))
def Nystroem_FMAPGenration(Data, n_components= 50, gamma = 2., kernel='rbf', random_state=0): from sklearn.kernel_approximation import Nystroem NystInt = Nystroem(kernel='rbf', gamma=gamma, n_components=n_components, random_state=random_state) Nyst_Sampler = NystInt.fit(Data) return Nyst_Sampler
del lka, lowrank_feats # # # # # # # # # # # # # # # # # Nyström again! # # # # # # # # # # # # # # # # print("Running Nystroem Approximation") factor = np.max(np.linalg.norm(train_features, axis=1)) train_features /= factor test_features /= factor dim = min(train_features.shape[1] * 2, train_features.shape[0]) print("Nystroem dim is {}".format(dim)) # Use the Nyström approximation in sklearn approx = Nystroem(kernel='rbf', gamma=1., n_components=dim) approx.fit(train_features) train_features = approx.transform(train_features) test_features = approx.transform(test_features) # # # # # # # # # # # # # # # # # Ridge regression with cross validation # # # # # # # # # # # # # # # # style = 'c' if train_features.shape[0] > train_features.shape[1] else 'k' clf = GridSearchCV(RidgeRegression(), { 'alpha': [(10**i) for i in range(-7, 0)], 'style': [style] }, n_jobs=4) clf.fit(train_features, train_onehot)
XtrainT = kcca.transform(ktrain) XtestT = kcca.transform(ktest) kccaScores = np.zeros((2,np.alen(nComponents))) for i,n in enumerate(nComponents): kccaScores[:,i] = util.classify(XtrainT[:,0:n],XtestT[:,0:n],labelsTrain,labelsTest) #%% Subsampling methods kpls = PLSRegression(n_components=150) nComponents = np.arange(173,2173,100) # Nystroem method elapTimeNys = np.zeros(np.shape(nComponents)) kplsScoresNys = np.zeros((2,3)) for i,n in enumerate(nComponents): nys = Nystroem(n_components=n,gamma=gamma) nys.fit(Xtrain) ktrain = nys.transform(Xtrain) ktest = nys.transform(Xtest) startTime = timeit.default_timer() kpls.fit(ktrain,Ytrain) elapTimeNys[i] = timeit.default_timer() - startTime XtrainT = kpls.transform(ktrain) XtestT = kpls.transform(ktest) if n==573: kplsScoresNys[:,0] = util.classify(XtrainT,XtestT,labelsTrain,labelsTest) elif n==1073: kplsScoresNys[:,1] = util.classify(XtrainT,XtestT,labelsTrain,labelsTest) elif n==1573: kplsScoresNys[:,2] = util.classify(XtrainT,XtestT,labelsTrain,labelsTest)
rbf2 = RBFSampler2(gamma=gamma, n_components=nc / 2) rbf2.fit(Xtr) # the random state needs to be specified so that the test and train features are the same clfa2 = svm.LinearSVC(C=C, loss='hinge') clfa2.fit(rbf2.transform(Xtr), Ytr) nys = Nystroem(kernel='rbf', gamma=gamma, coef0=1, degree=3, kernel_params=None, n_components=nc, random_state=None) nys.fit(Xtr) clf3 = svm.LinearSVC(C=C, loss='hinge') clf3.fit(nys.transform(Xtr), Ytr) clf = svm.SVC(kernel='rbf', C=C, gamma=gamma) clf.fit(Xtr, Ytr) plt.close('all') npts = 50 xm, xM = np.min(Xtr), np.max(Xtr) x = np.linspace(xm, xM, npts) y = np.linspace(xm, xM, npts) t = np.array(list(itertools.product(x, y))) def plotit(z, title):
XtestT = kcca.transform(ktest) kccaScores = np.zeros((2, np.alen(nComponents))) for i, n in enumerate(nComponents): kccaScores[:, i] = util.classify(XtrainT[:, 0:n], XtestT[:, 0:n], labelsTrain, labelsTest) #%% Subsampling methods kpls = PLSRegression(n_components=150) nComponents = np.arange(173, 2173, 100) # Nystroem method elapTimeNys = np.zeros(np.shape(nComponents)) kplsScoresNys = np.zeros((2, 3)) for i, n in enumerate(nComponents): nys = Nystroem(n_components=n, gamma=gamma) nys.fit(Xtrain) ktrain = nys.transform(Xtrain) ktest = nys.transform(Xtest) startTime = timeit.default_timer() kpls.fit(ktrain, Ytrain) elapTimeNys[i] = timeit.default_timer() - startTime XtrainT = kpls.transform(ktrain) XtestT = kpls.transform(ktest) if n == 573: kplsScoresNys[:, 0] = util.classify(XtrainT, XtestT, labelsTrain, labelsTest) elif n == 1073: kplsScoresNys[:, 1] = util.classify(XtrainT, XtestT, labelsTrain, labelsTest) elif n == 1573:
class NystronemSampler(Transformer): def __init__(self, kernel='rbf', n_components=100, gamma=1.0, degree=3, coef0=1, random_state=None): super().__init__("nystronem_sampler", 15, random_state=random_state) self.input_type = [NUMERICAL, DISCRETE, CATEGORICAL] self.compound_mode = 'only_new' self.output_type = NUMERICAL self.kernel = kernel self.n_components = n_components self.gamma = gamma self.degree = degree self.coef0 = coef0 self.random_state = random_state @ease_trans def operate(self, input_datanode, target_fields=None): X, y = input_datanode.data X_new = X[:, target_fields].astype(np.float64) # Because the pipeline guarantees that each feature is positive, # clip all values below zero to zero if self.kernel == 'chi2': if scipy.sparse.issparse(X_new): X_new.data[X_new.data < 0] = 0.0 else: X_new[X_new < 0] = 0.0 if not self.model: n_components = min(X.shape[0], self.n_components) self.gamma = float(self.gamma) self.degree = int(self.degree) self.coef0 = float(self.coef0) self.model = Nystroem(kernel=self.kernel, n_components=n_components, gamma=self.gamma, degree=self.degree, coef0=self.coef0, random_state=self.random_state) self.model.fit(X_new.astype(np.float64)) _X = self.model.transform(X_new) return _X @staticmethod def get_hyperparameter_search_space(dataset_properties=None): if dataset_properties is not None and \ (dataset_properties.get("sparse") is True or dataset_properties.get("signed") is False): allow_chi2 = False else: allow_chi2 = True possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine'] if allow_chi2: possible_kernels.append("chi2") kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf') n_components = UniformIntegerHyperparameter("n_components", 10, 2000, default_value=100, log=True) gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8, log=True, default_value=0.1) degree = UniformIntegerHyperparameter('degree', 2, 5, 3) coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0) cs = ConfigurationSpace() cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components]) degree_depends_on_poly = EqualsCondition(degree, kernel, "poly") coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"]) gamma_kernels = ["poly", "rbf", "sigmoid"] if allow_chi2: gamma_kernels.append("chi2") gamma_condition = InCondition(gamma, kernel, gamma_kernels) cs.add_conditions( [degree_depends_on_poly, coef0_condition, gamma_condition]) return cs
def estimateAll(self, loggedData): numInstances = len(loggedData) targets = numpy.zeros(numInstances, order='C', dtype=numpy.float64) null_covariates = scipy.sparse.lil_matrix( (numInstances, self.numFeatures * self.rankingSize), dtype=numpy.float64) target_covariates = scipy.sparse.lil_matrix( (numInstances, self.numFeatures * self.rankingSize), dtype=numpy.float64) print("Starting to create covariates", flush=True) for j in range(numInstances): currentDatapoint = loggedData[j] targets[j] = currentDatapoint[2] currentQuery = currentDatapoint[0] currentRanking = currentDatapoint[1] newRanking = currentDatapoint[3] nullFeatures = self.loggingPolicy.dataset.features[currentQuery][ currentRanking, :] nullFeatures.eliminate_zeros() targetFeatures = self.loggingPolicy.dataset.features[currentQuery][ newRanking, :] targetFeatures.eliminate_zeros() null_covariates.data[j] = nullFeatures.data target_covariates.data[j] = targetFeatures.data nullIndices = nullFeatures.indices targetIndices = targetFeatures.indices for k in range(nullFeatures.shape[0]): nullIndices[nullFeatures.indptr[k]:nullFeatures. indptr[k + 1]] += k * self.numFeatures targetIndices[targetFeatures.indptr[k]:targetFeatures. indptr[k + 1]] += k * self.numFeatures null_covariates.rows[j] = nullIndices target_covariates.rows[j] = targetIndices if j % 1000 == 0: print(".", end='', flush=True) del currentDatapoint del nullFeatures del targetFeatures print("Converting covariates", flush=True) null_covariates = null_covariates.toarray() target_covariates = target_covariates.toarray() scaler = sklearn.preprocessing.MinMaxScaler() scaler.fit(null_covariates) s_null_covariates = scaler.transform(null_covariates) s_target_covariates = scaler.transform(target_covariates) print("Finished conversion", flush=True) print("Calculating heuristic kernel param", flush=True) random_indices = numpy.arange( 0, s_null_covariates.shape[0]) # array of all indices numpy.random.shuffle(random_indices) # shuffle the array sample_null_covar = s_null_covariates[ random_indices[:2000]] # get N samples without replacement sample_target_covar = s_target_covariates[ random_indices[:2000]] # get N samples without replacement recom_param = (0.5 * self.kernel_param) / numpy.median( pdist(numpy.vstack([sample_null_covar, sample_target_covar]), 'sqeuclidean')) print("Computing kernel matrix", flush=True) m = null_covariates.shape[0] n = target_covariates.shape[0] reg_params = self.reg_param / n if self.approx and m > self.p: p = self.p rets = [] for i in range(self.n_approx): nystroem = Nystroem(gamma=recom_param, n_components=p) nystroem.fit(s_null_covariates) nullPhi = nystroem.transform(s_null_covariates) targetPhi = nystroem.transform(s_target_covariates) b = numpy.dot(targetPhi.T, numpy.repeat(1.0 / m, m, axis=0)) A = nullPhi.T.dot(nullPhi) + numpy.diag( numpy.repeat(p * reg_params, p)) beta_vec_approx = nullPhi.dot( scipy.sparse.linalg.cg(A, b, tol=1e-08, maxiter=5000)[0]) ret = numpy.dot(beta_vec_approx, targets) / beta_vec_approx.sum() rets.append(ret) return numpy.mean(rets) else: nullRecomMatrix = self.kernel(s_null_covariates, s_null_covariates, recom_param) targetRecomMatrix = self.kernel(s_null_covariates, s_target_covariates, recom_param) b = numpy.dot(targetRecomMatrix, numpy.repeat(1.0 / m, m, axis=0)) A = nullRecomMatrix + numpy.diag(numpy.repeat(n * reg_params, n)) print("Finding beta_vec", flush=True) beta_vec, _ = scipy.sparse.linalg.cg(A, b, tol=1e-08, maxiter=5000) return numpy.dot(beta_vec, targets) / beta_vec.sum()