def test_nystroem_callable():
# Test Nystroem on a callable.
rnd = np.random.RandomState(42)
n_samples = 10
X = rnd.uniform(size=(n_samples, 4))
def logging_histogram_kernel(x, y, log):
"""Histogram kernel that writes to a log."""
log.append(1)
return np.minimum(x, y).sum()
kernel_log = []
X = list(X) # test input validation
Nystroem(kernel=logging_histogram_kernel,
n_components=(n_samples - 1),
kernel_params={'log': kernel_log}).fit(X)
assert len(kernel_log) == n_samples * (n_samples - 1) / 2
# if degree, gamma or coef0 is passed, we raise a warning
msg = "Don't pass gamma, coef0 or degree to Nystroem"
params = ({'gamma': 1}, {'coef0': 1}, {'degree': 2})
for param in params:
ny = Nystroem(kernel=_linear_kernel, **param)
with pytest.raises(ValueError, match=msg):
ny.fit(X)
def test_nystroem_approximation():
# some basic tests
rnd = np.random.RandomState(0)
X = rnd.uniform(size=(10, 4))
# With n_components = n_samples this is exact
X_transformed = Nystroem(n_components=X.shape[0]).fit_transform(X)
K = rbf_kernel(X)
assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)
trans = Nystroem(n_components=2, random_state=rnd)
X_transformed = trans.fit(X).transform(X)
assert_equal(X_transformed.shape, (X.shape[0], 2))
# test callable kernel
linear_kernel = lambda X, Y: np.dot(X, Y.T)
trans = Nystroem(n_components=2, kernel=linear_kernel, random_state=rnd)
X_transformed = trans.fit(X).transform(X)
assert_equal(X_transformed.shape, (X.shape[0], 2))
# test that available kernels fit and transform
kernels_available = kernel_metrics()
for kern in kernels_available:
trans = Nystroem(n_components=2, kernel=kern, random_state=rnd)
X_transformed = trans.fit(X).transform(X)
assert_equal(X_transformed.shape, (X.shape[0], 2))
class NystroemImpl():
def __init__(self,
kernel='rbf',
gamma=None,
coef0=None,
degree=None,
kernel_params=None,
n_components=100,
random_state=None):
self._hyperparams = {
'kernel': kernel,
'gamma': gamma,
'coef0': coef0,
'degree': degree,
'kernel_params': kernel_params,
'n_components': n_components,
'random_state': random_state
}
self._wrapped_model = Op(**self._hyperparams)
def fit(self, X, y=None):
if (y is not None):
self._wrapped_model.fit(X, y)
else:
self._wrapped_model.fit(X)
return self
def transform(self, X):
return self._wrapped_model.transform(X)
def test_nystroem_callable():
# Test Nystroem on a callable.
rnd = np.random.RandomState(42)
n_samples = 10
X = rnd.uniform(size=(n_samples, 4))
def logging_histogram_kernel(x, y, log):
"""Histogram kernel that writes to a log."""
log.append(1)
return np.minimum(x, y).sum()
kernel_log = []
X = list(X) # test input validation
Nystroem(kernel=logging_histogram_kernel,
n_components=(n_samples - 1),
kernel_params={'log': kernel_log}).fit(X)
assert_equal(len(kernel_log), n_samples * (n_samples - 1) / 2)
def linear_kernel(X, Y):
return np.dot(X, Y.T)
# if degree, gamma or coef0 is passed, we raise a warning
msg = "Don't pass gamma, coef0 or degree to Nystroem"
params = ({'gamma': 1}, {'coef0': 1}, {'degree': 2})
for param in params:
ny = Nystroem(kernel=linear_kernel, **param)
with pytest.raises(ValueError, match=msg):
ny.fit(X)
class Kernel_tica(object):
def __init__(
self,
n_components,
lag_time,
gamma, # gamma value for rbf kernel
n_components_nystroem=100, # number of components for Nystroem kernel approximation
landmarks=None,
shrinkage=None,
weights='empirical' # if 'koopman', use Koopman reweighting for tICA (see Wu, Hao, et al. "Variational Koopman models: slow collective variables and molecular kinetics from short off-equilibrium simulations." The Journal of Chemical Physics 146.15 (2017): 154104.)
):
self._n_components = n_components
self._lag_time = lag_time
self._n_components_nystroem = n_components_nystroem
self._landmarks = landmarks
self._gamma = gamma
self._nystroem = Nystroem(gamma=gamma,
n_components=n_components_nystroem)
self._weights = weights
# self._tica = tICA(n_components=n_components, lag_time=lag_time, shrinkage=shrinkage)
self._shrinkage = shrinkage
return
def fit(self, sequence_list):
if self._landmarks is None:
self._nystroem.fit(np.concatenate(sequence_list))
else:
print("using landmarks")
self._nystroem.fit(self._landmarks)
sequence_transformed = [
self._nystroem.transform(item) for item in sequence_list
]
# define tica object at fit() with sequence_list supplied for initialization, as it is required by
# Koopman reweighting
self._tica = py.coordinates.tica(sequence_transformed,
lag=self._lag_time,
dim=self._n_components,
kinetic_map=True,
weights=self._weights)
return
def transform(self, sequence_list):
return self._tica.transform(
[self._nystroem.transform(item) for item in sequence_list])
def fit_transform(self, sequence_list):
self.fit(sequence_list)
return self.transform(sequence_list)
def score(self, sequence_list):
model = self.__class__(
n_components=self._n_components,
lag_time=self._lag_time,
gamma=self._gamma,
n_components_nystroem=self._n_components_nystroem,
landmarks=self._landmarks,
shrinkage=self._shrinkage)
model.fit(sequence_list)
return np.sum(model._tica.eigenvalues)
def init_scaler(self, observed_cmd, gamma=0.5):
self.cmd_scaler = MinMaxScaler()
self.cmd_scaler.fit(observed_cmd)
scaled_observed_cmd = self.cmd_scaler.transform(observed_cmd)
Phi_approx = Nystroem(kernel='rbf', n_components=50, gamma=gamma)
Phi_approx.fit(scaled_observed_cmd)
self.mapping = Phi_approx.transform
print('scaler initialized and mapping defined!')
return scaled_observed_cmd
def test_nystroem_component_indices():
"""Check that `component_indices_` corresponds to the subset of
training points used to construct the feature map.
Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/20474
"""
X, _ = make_classification(n_samples=100, n_features=20)
feature_map_nystroem = Nystroem(
n_components=10,
random_state=0,
)
feature_map_nystroem.fit(X)
assert feature_map_nystroem.component_indices_.shape == (10, )
class NystroemImpl:
def __init__(self, **hyperparams):
self._hyperparams = hyperparams
self._wrapped_model = Op(**self._hyperparams)
def fit(self, X, y=None):
if y is not None:
self._wrapped_model.fit(X, y)
else:
self._wrapped_model.fit(X)
return self
def transform(self, X):
return self._wrapped_model.transform(X)
def test_nystroem_precomputed_kernel():
# Non-regression: test Nystroem on precomputed kernel.
# PR - 14706
rnd = np.random.RandomState(12)
X = rnd.uniform(size=(10, 4))
K = polynomial_kernel(X, degree=2, coef0=0.1)
nystroem = Nystroem(kernel="precomputed", n_components=X.shape[0])
X_transformed = nystroem.fit_transform(K)
assert_array_almost_equal(np.dot(X_transformed, X_transformed.T), K)
# if degree, gamma or coef0 is passed, we raise a ValueError
msg = "Don't pass gamma, coef0 or degree to Nystroem"
params = ({"gamma": 1}, {"coef0": 1}, {"degree": 2})
for param in params:
ny = Nystroem(kernel="precomputed", n_components=X.shape[0], **param)
with pytest.raises(ValueError, match=msg):
ny.fit(K)
def kernel_transformation_using_nystroem_rbf(self,df,cat):
df=df.fillna(0)
df=df.replace([np.inf, -np.inf], 0)
datecol=[x for x in df.columns if df[x].dtypes=='datetime64[ns]']
X1=[x for x in df.columns if df[x].dtypes != 'object' and x not in datecol and x not in self.target]
X=[x for x in X1 if x not in cat]
y=self.target
j = np.linspace((10**-2),(10**2),50)
g=0
max1=0
df=df.fillna(0)
df=df.replace(np.inf, 0)
df=df.replace(-np.inf, 0)
for i in j:
rbf_feature = Nystroem(kernel = 'rbf', gamma=i, random_state=2,n_components=10)
rbf_feature.fit(df[X])
X_features = rbf_feature.transform(df[X])
X_features=np.nan_to_num(X_features)
# SGDClassifier(loss='hinge', penalty='l2', alpha=0.0001, l1_ratio=0.15, fit_intercept=True, shuffle=True, verbose=0, epsilon=0.1, n_jobs=1, random_state=None, learning_rate='optimal', eta0=0.0, power_t=0.5, class_weight=None, warm_start=False, average=False, n_iter=None)
clf = SGDClassifier()
clf.fit(X_features, df[y])
y_pred = clf.predict(X_features)
score=f1_score(df[y], y_pred, average='micro')
if(score>max1):
max1=score
g=i
rbf_feature = RBFSampler(gamma=g, random_state=2,n_components=10)
rbf_feature.fit(df[X])
X_features = rbf_feature.transform(df[X])
l=[]
for r in range(10):
l.append('k_'+str(r))
X_features=pd.DataFrame(data=X_features,columns=l)
# SGDClassifier(loss='hinge', penalty='l2', alpha=0.0001, l1_ratio=0.15, fit_intercept=True,shuffle=True, verbose=0, epsilon=0.1, n_jobs=1, random_state=None, learning_rate='optimal', eta0=0.0, power_t=0.5, class_weight=None, warm_start=False, average=False, n_iter=None)
clf = SGDClassifier()
clf.fit(X_features, df[y])
score=f1_score(df[y], y_pred, average='micro')
print("Score is")
print(score)
print(g)
return X_features
class NystromScikit:
"""
Nystrom implementation form Scikit Learn wrapper.
The main difference is in selection of inducing inputs.
"""
def __init__(self, rank=10, random_state=42):
"""
:param rank: (``int``) Maximal decomposition rank.
:param random_state: (``int``) Random generator seed.
"""
self.trained = False
self.rank = rank
self.random_state = random_state
def fit(self, K, y):
"""
Fit approximation to the kernel function / matrix.
:param K: (``numpy.ndarray``) or of (``Kinterface``). The kernel to be approximated with G.
:param y: (``numpy.ndarray``) Class labels :math:`y_i \in {-1, 1}` or regression targets.
"""
assert isinstance(K, Kinterface)
self.n = K.shape[0]
kernel = lambda x, y: K.kernel(x, y, **K.kernel_args)
self.model = Nystroem(kernel=kernel,
n_components=self.rank,
random_state=self.random_state)
self.model.fit(K.data, y)
self.active_set_ = list(self.model.component_indices_[:self.rank])
assert len(set(self.active_set_)) == len(self.active_set_) == self.rank
R = self.model.normalization_
self.G = K[:, self.active_set_].dot(R)
self.trained = True
def nystroem_pca(X, nystroem_kwargs, pca_kwargs, verbose=1):
"""
Perform Nystroem kernel approximation and then PCA decomposition. The Nystroem method constructs an
approximate feature map, for an arbitrary kernel using a subset of the data as basis. We then
apply this feature map to X, and perform PCA decomposition in the new feature space.
:param X: Values of our data
:param nystroem_kwargs: (dict) Keyword arguments that are passed to the constructor of an implementation
of the Nystroem method - `sklearn.kernel_approximation.Nystroem`.
:param pca_kwargs: dict) Keyword arguments that are passed to `sklearn.decomposition.PCA`.
:param verbose: (int) If verbose=1, print information relevant to the PCA decomposition. Set verbose=0
for no output.
:return: `numpy.ndarray`: The PCA decomposition of our data set in the new feature space, that was
approximated via the nystroem method.
"""
# Get the nystroem approximation
nystroem = Nystroem(**nystroem_kwargs)
nystroem.fit(X)
X_nystroem = nystroem.transform(X)
# Get the PCA decomposition
pca_nystroem = PCA(**pca_kwargs)
pca_nystroem.fit(X_nystroem)
X_nystroem_pca = pca_nystroem.transform(X_nystroem)
# Print Relevant PCA information
if verbose:
print("Dimensionality reduction: {} -> {}".format(
X_nystroem.shape[1], X_nystroem_pca.shape[1]))
print("Variance explained by each component:",
(pca_nystroem.explained_variance_ratio_ * 1000).astype(int) /
1000)
print(
"Total variance explained by those {} components:".format(
X_nystroem_pca.shape[1]),
format(pca_nystroem.explained_variance_ratio_.sum(), ".4f"))
return X_nystroem_pca
def main():
data = pd.DataFrame(circle(3000), columns=['x', 'y'])
ns = Nystroem(n_components=3000, gamma=0.15)
ns.fit(data)
fullsample = pd.DataFrame(
[(x, y) for x in np.around(
np.linspace(-20.0, 20.0, 128),
decimals=2,
) for y in np.around(
np.linspace(-20.0, 20.0, 128),
decimals=2,
)],
columns=['x', 'y'],
)
transformed = ns.transform(fullsample)
fullsample['c'] = pd.Series(transformed[:, 0])
sns.heatmap(fullsample.pivot('x', 'y', 'c'),
xticklabels=32,
yticklabels=32)
pyplot.show()
class NystromScikit:
"""
Nystrom implementation form Scikit Learn wrapper.
The main difference is in selection of inducing inputs.
"""
def __init__(self, rank=10, random_state=42):
"""
:param rank: (``int``) Maximal decomposition rank.
:param random_state: (``int``) Random generator seed.
"""
self.trained = False
self.rank = rank
self.random_state = random_state
def fit(self, K, y):
"""
Fit approximation to the kernel function / matrix.
:param K: (``numpy.ndarray``) or of (``Kinterface``). The kernel to be approximated with G.
:param y: (``numpy.ndarray``) Class labels :math:`y_i \in {-1, 1}` or regression targets.
"""
assert isinstance(K, Kinterface)
self.n = K.shape[0]
kernel = lambda x, y: K.kernel(x, y, **K.kernel_args)
self.model = Nystroem(kernel=kernel,
n_components=self.rank,
random_state=self.random_state)
self.model.fit(K.data, y)
self.active_set_ = list(self.model.component_indices_[:self.rank])
assert len(set(self.active_set_)) == len(self.active_set_) == self.rank
R = self.model.normalization_
self.G = K[:, self.active_set_].dot(R)
self.trained = True
def construct_Q(self, feature_normalization=True):
""" Step 1 and 2 in Algorithm 1 in MSc. Thesis.
Computes Q, such that QQ^T \approx K, where K is the RBF kernel matrix.
Also applies normalization to the features by default. This is carried
over by example from the CCNN code of Zhang et. al
Input
Z_train, Z_test: (N,P,d1) arrays. Each Z[i,:,:] is one Z(x_i).
Result from __init__.
Output
Q_train, Q_test: (N,P,m) arrays Each Q[i,:,:] is one Q(x_i).
Used in train() function below.
"""
from sklearn.kernel_approximation import Nystroem
import numpy as np
import math
tprint("Using Scikitlearn Nystroem function")
tprint("Creating Q...")
Z_train = self.Z[0:self.n_train].reshape(
(self.n_train * self.P, self.d1))
Z_test = self.Z[self.n_train:self.n].reshape(
(self.n_test * self.P, self.d1))
transformer = Nystroem(gamma=self.gamma, n_components=self.nystrom_dim)
transformer = transformer.fit(X=Z_train)
Q_train = transformer.transform(Z_train)
Q_test = transformer.transform(Z_test)
self.Q_train = Q_train.reshape(
(self.n_train, self.P, self.nystrom_dim))
self.Q_test = Q_test.reshape((self.n_test, self.P, self.nystrom_dim))
if feature_normalization == True:
self.Q_train = self.Q_train.reshape(
(self.n_train * self.P, self.nystrom_dim))
self.Q_train -= np.mean(self.Q_train, axis=0)
self.Q_train /= LA.norm(self.Q_train) / math.sqrt(
self.n_train * self.P)
self.Q_train = self.Q_train.reshape(
(self.n_train, self.P, self.nystrom_dim))
self.Q_test = self.Q_test.reshape(
(self.n_test * self.P, self.nystrom_dim))
self.Q_test -= np.mean(self.Q_test, axis=0)
self.Q_test /= LA.norm(self.Q_test) / math.sqrt(
self.n_train * self.P)
self.Q_test = self.Q_test.reshape(
(self.n_test, self.P, self.nystrom_dim))
from params import ts_depths,n_fea,gamma, np, sp
class Whitener:
def __init__(self,X):
self.Xmean = X.mean(0)
self.Xstd = X.std(0)
def whiten(self,Z):
return (Z-self.Xmean)/self.Xstd
def unwhiten(self,Zw):
return Zw*self.Xstd + self.Xmean
def expkern(x,y):
return np.exp(-gamma*la.norm(x-y))
wh = Whitener(ts_depths)
ts_depths_w = wh.whiten(ts_depths)
xx = np.linspace(ts_depths_w.min(),ts_depths_w.max(),n_fea)[:,np.newaxis]
rbf_tr = Nystroem(expkern,gamma,n_components=n_fea)
#rbf_tr = Nystroem(gamma=gamma,n_components=n_fea)
#class rbf_transformer:
# def __init__(self,X,gamma):
# self.X = X
# self.gamma = gamma
# def transform(self,xx):
# return rbf_kernel(xx,self.X)
#rbf_tr = rbf_transformer(xx,gamma)
#rbf_tr.fit(x)
rbf_tr.fit(xx)
ts_depths_tr = rbf_tr.transform(ts_depths_w)
dataTrainT = kcca.transform(ktrain)
dataTestT = kcca.transform(ktest)
kccaScores = np.zeros((2,np.alen(nComponents)))
for i,n in enumerate(nComponents):
kccaScores[:,i] = util.classify(dataTrainT[:,0:n],dataTestT[:,0:n],labelsTrain,labelsTest)
#%% Subsampling methods
kpls = PLSRegression(n_components=150)
nComponents = np.arange(173,2173,100)
# Nystroem method
elapTimeNys = np.zeros(np.shape(nComponents))
kplsScoresNys = np.zeros((2,3))
for i,n in enumerate(nComponents):
nys = Nystroem(n_components=n,gamma=gamma)
nys.fit(dataTrain)
ktrain = nys.transform(dataTrain)
ktest = nys.transform(dataTest)
startTime = timeit.default_timer()
kpls.fit(ktrain,Ytrain)
elapTimeNys[i] = timeit.default_timer() - startTime
dataTrainT = kpls.transform(ktrain)
dataTestT = kpls.transform(ktest)
if n==573:
kplsScoresNys[:,0] = util.classify(dataTrainT,dataTestT,labelsTrain,labelsTest)
elif n==1073:
kplsScoresNys[:,1] = util.classify(dataTrainT,dataTestT,labelsTrain,labelsTest)
elif n==1573:
kplsScoresNys[:,2] = util.classify(dataTrainT,dataTestT,labelsTrain,labelsTest)
class ParametricModelApproximation(object):
"""Approximate a Gaussian Process by a parametric model.
Approximating a Gaussian Process by a parametric model can be useful if
one has to evaluate a sample function from the GP repeatedly or on many
evaluation points as this would become computationally very expensive
with a GP.
Parameters
----------
model : GaussianProcessRegressor
The Gaussian Process which is to be approximated
bounds: list of pair of floats
The boundaries of the data space. This is used when determining the
features of the parametric approximation (they are centered at random
points in the data space)
n_components: int
The number of features/parameters of the parametric model
seed: int
The seed of the random number generator
"""
def __init__(self, model, bounds, n_components, seed):
self.gp = model
self.bounds = bounds
self.n_components = n_components
self.rng = np.random.RandomState(seed)
self.X_space = self.rng.uniform(self.bounds[:, 0], self.bounds[:, 1],
(1000, self.bounds.shape[0]))
assert self.gp.X_fit_.shape[1] == self.X_space.shape[1]
self.kernel = self.gp.kernel_
self.nystr = Nystroem(
n_components=min(self.n_components, self.X_space.shape[0]),
kernel='precomputed', random_state=self.rng)
self.nystr.fit(self.kernel(self.X_space))
def determine_coefs(self, X_query=None, y_query_samples=None, n_samples=1):
""" Determine coefficients of parametric model.
Simulate an evaluation at X_query with outcomes y_query_samples.
Determine coefficients of parametric model the updated GP.
Parameters
----------
X_query : ndarray-like, default: None
The query point at which an additional evaluation is simulated.
If None, a parametric approximation of the unmodified GP is
returned.
y_query_samples: ndarray-like, default: None
The possible outcomes of a query at X_query.
n_samples: int
The number of independent samples of model coefficients from the
Bayesian posterior over model coefficients
"""
if X_query is not None:
X_query = np.asarray(X_query)
X_queried = np.vstack((self.gp.X_fit_, X_query))
else:
X_queried = self.gp.X_fit_
y_queried = self.gp.y_fit_
Phi = self.nystr.transform(self.kernel(self.X_space, X_queried))
A = Phi.T.dot(Phi) + self.gp.alpha * np.eye(Phi.shape[1])
A_inv = np.linalg.inv(A)
cov = self.gp.alpha * A_inv
coefs = \
np.empty((n_samples, self.n_components, y_query_samples.shape[0]))
for i in range(y_query_samples.shape[0]): # XXX: Vectorize
y_queried = np.hstack((self.gp.y_fit_, y_query_samples[i]))
mean = A_inv.dot(Phi.T).dot(y_queried)
coefs[:, :, i] = self.rng.multivariate_normal(mean, cov, n_samples)
return np.array(coefs)
def __call__(self, X, coefs):
""" Evaluate parametric model at X for the given sampled coefficients.
Parameters
----------
X : ndarray-like
The points at which the parametric model is to be evaluated
coefs: ndarray-like
The coefficients of the parametric model.
"""
X = np.atleast_2d(X)
Phi = self.nystr.transform(self.kernel(self.X_space, X))
f = Phi.dot(coefs)
return f
width=1)))
fig.append_trace(projection_fourier_1, 1, 2)
fig.append_trace(projection_fourier_2, 1, 2)
fig['layout']['xaxis2'].update(title='1st principal component',
zeroline=False,
showgrid=False)
fig['layout']['yaxis2'].update(title='2nd component',
zeroline=False,
showgrid=False)
## Nystroem
nystroem = Nystroem(gamma=gamma, random_state=1)
nystroem.fit(X_pca)
X_nystroem_pca = nystroem.transform(X_pca)
projection_nystroem_1 = go.Scatter(x=X_nystroem_pca[reds, 0],
y=X_nystroem_pca[reds, 1],
mode='markers',
showlegend=False,
marker=dict(color='red',
line=dict(color='black',
width=1)))
projection_nystroem_2 = go.Scatter(x=X_nystroem_pca[blues, 0],
y=X_nystroem_pca[blues, 1],
mode='markers',
showlegend=False,
marker=dict(color='blue',
line=dict(color='black',
def make_nystrom_evaluation(x_train, y_train, x_test, y_test, U_centroids):
"""
Evaluation Nystrom construction time and approximation precision.
The approximation is based on a subsample of size n_sample of the input data set.
:param x_train: Input dataset as ndarray.
:param U_centroids: The matrix of centroids as ndarray or SparseFactor object
:param n_sample: The number of sample to take into account in the reconstruction (can't be too large)
:return:
"""
n_sample = paraman["--nystrom"]
if n_sample > x_train.shape[0]:
logger.warning(
"Batch size for nystrom evaluation is bigger than data size. {} > {}. Using "
"data size instead.".format(n_sample, x_train.shape[0]))
n_sample = x_train.shape[0]
paraman["--nystrom"] = n_sample
# Compute euristic gamma as the mean of euclidian distance between example
gamma = compute_euristic_gamma(x_train)
log_memory_usage(
"Memory after euristic gamma computation in make_nystrom_evaluation")
# precompute the centroids norm for later use (optimization)
centroids_norm = get_squared_froebenius_norm_line_wise(U_centroids)
# centroids_norm = None
indexes_samples = np.random.permutation(x_train.shape[0])[:n_sample]
sample = x_train[indexes_samples]
samples_norm = None
log_memory_usage(
"Memory after sample selection in make_nystrom_evaluation")
########################
# Nystrom on centroids #
########################
logger.info("Build Nystrom on centroids")
## TIME: nystrom build time
# nystrom build time is Nystrom preparation time for later use.
## START
nystrom_build_start_time = time.process_time()
metric = prepare_nystrom(U_centroids, centroids_norm, gamma=gamma)
nystrom_build_stop_time = time.process_time()
log_memory_usage("Memory after SVD computation in make_nystrom_evaluation")
# STOP
nystrom_build_time = nystrom_build_stop_time - nystrom_build_start_time
## TIME: nystrom inference time
# Nystrom inference time is the time for Nystrom transformation for all the samples.
## START
nystrom_inference_time_start = time.process_time()
nystrom_embedding = nystrom_transformation(sample,
U_centroids,
metric,
centroids_norm,
samples_norm,
gamma=gamma)
nystrom_approx_kernel_value = nystrom_embedding @ nystrom_embedding.T
nystrom_inference_time_stop = time.process_time()
log_memory_usage(
"Memory after kernel matrix approximation in make_nystrom_evaluation")
## STOP
nystrom_inference_time = (nystrom_inference_time_stop -
nystrom_inference_time_start) / n_sample
################################################################
######################
# Nystrom on uniform #
######################
logger.info("Build Nystrom on uniform sampling")
indexes_uniform_samples = np.random.permutation(
x_train.shape[0])[:U_centroids.shape[0]]
uniform_sample = x_train[indexes_uniform_samples]
uniform_sample_norm = None
log_memory_usage(
"Memory after uniform sample selection in make_nystrom_evaluation")
metric_uniform = prepare_nystrom(uniform_sample,
uniform_sample_norm,
gamma=gamma)
log_memory_usage(
"Memory after SVD computation in uniform part of make_nystrom_evaluation"
)
nystrom_embedding_uniform = nystrom_transformation(sample,
uniform_sample,
metric_uniform,
uniform_sample_norm,
samples_norm,
gamma=gamma)
nystrom_approx_kernel_value_uniform = nystrom_embedding_uniform @ nystrom_embedding_uniform.T
#################################################################
###############
# Real Kernel #
###############
logger.info("Compute real kernel matrix")
real_kernel_special = special_rbf_kernel(sample,
sample,
gamma,
norm_X=samples_norm,
norm_Y=samples_norm)
real_kernel = rbf_kernel(sample, sample, gamma)
real_kernel_norm = np.linalg.norm(real_kernel)
log_memory_usage(
"Memory after real kernel computation in make_nystrom_evaluation")
#################################
# Sklearn based Nystrom uniform #
#################################
sklearn_nystrom = Nystroem(gamma=gamma,
n_components=uniform_sample.shape[0])
sklearn_nystrom = sklearn_nystrom.fit(uniform_sample)
sklearn_transfo = sklearn_nystrom.transform(sample)
kernel_sklearn_nys = sklearn_transfo @ sklearn_transfo.T
################################################################
####################
# Error evaluation #
####################
sampled_froebenius_norm = np.linalg.norm(nystrom_approx_kernel_value -
real_kernel) / real_kernel_norm
sampled_froebenius_norm_uniform = np.linalg.norm(
nystrom_approx_kernel_value_uniform - real_kernel) / real_kernel_norm
# svm evaluation
if x_test is not None:
logger.info("Start classification")
time_classification_start = time.process_time()
x_train_nystrom_embedding = nystrom_transformation(x_train,
U_centroids,
metric,
centroids_norm,
None,
gamma=gamma)
x_test_nystrom_embedding = nystrom_transformation(x_test,
U_centroids,
metric,
centroids_norm,
None,
gamma=gamma)
linear_svc_clf = LinearSVC()
linear_svc_clf.fit(x_train_nystrom_embedding, y_train)
accuracy_nystrom_svm = linear_svc_clf.score(x_test_nystrom_embedding,
y_test)
time_classification_stop = time.process_time()
delta_time_classification = time_classification_stop - time_classification_start
else:
accuracy_nystrom_svm = None
delta_time_classification = None
nystrom_results = {
"nystrom_build_time": nystrom_build_time,
"nystrom_inference_time": nystrom_inference_time,
"nystrom_sampled_error_reconstruction": sampled_froebenius_norm,
"nystrom_sampled_error_reconstruction_uniform":
sampled_froebenius_norm_uniform,
"nystrom_svm_accuracy": accuracy_nystrom_svm,
"nystrom_svm_time": delta_time_classification
}
resprinter.add(nystrom_results)
def kernel_approximation(self, x, param_info):
#kapp = RBFSampler(gamma=param_info.kapp_gamma, param_info.kapp_num)
kapp = Nystroem(gamma=param_info.kapp_gamma,
n_components=param_info.kapp_num)
self.learned_kapp = kapp.fit(x)
class KernelLogisticRegression(LogisticRegression):
"""A simple kernel logistic implementation using a Nystroem kernel approximation
Warnings
--------
This kernel method is not specialized for temporal classification.
See Also
--------
wildboar.datasets.outlier.EmmottLabeler : Synthetic outlier dataset construction
"""
def __init__(self,
kernel=None,
*,
kernel_params=None,
n_components=100,
penalty="l2",
dual=False,
tol=1e-4,
C=1.0,
fit_intercept=True,
intercept_scaling=1,
class_weight=None,
random_state=None,
solver="lbfgs",
max_iter=100,
multi_class="auto",
verbose=0,
warm_start=False,
n_jobs=None,
l1_ratio=None):
"""Create a new kernel logistic regression
Parameters
----------
kernel : str, optional
The kernel function to use. See `sklearn.metrics.pairwise.kernel_metric` for kernels. The default kernel
is 'rbf'.
kernel_params : dict, optional
Parameters to the kernel function.
n_components : int, optional
Number of features to construct
"""
super().__init__(
penalty=penalty,
dual=dual,
tol=tol,
C=C,
fit_intercept=fit_intercept,
intercept_scaling=intercept_scaling,
class_weight=class_weight,
random_state=random_state,
solver=solver,
max_iter=max_iter,
multi_class=multi_class,
verbose=verbose,
warm_start=warm_start,
n_jobs=n_jobs,
l1_ratio=l1_ratio,
)
self.kernel = kernel
self.kernel_params = kernel_params
self.n_components = n_components
def fit(self, x, y, sample_weight=None):
random_state = check_random_state(self.random_state)
kernel = self.kernel or "rbf"
n_components = min(x.shape[0], self.n_components)
self.nystroem_ = Nystroem(
kernel=kernel,
kernel_params=self.kernel_params,
n_components=n_components,
random_state=random_state.randint(np.iinfo(np.int32).max),
)
self.nystroem_.fit(x)
super().fit(self.nystroem_.transform(x),
y,
sample_weight=sample_weight)
return self
def decision_function(self, x):
check_is_fitted(self)
return super().decision_function(self.nystroem_.transform(x))
def Nystroem_FMAPGenration(Data, n_components= 50, gamma = 2., kernel='rbf', random_state=0):
from sklearn.kernel_approximation import Nystroem
NystInt = Nystroem(kernel='rbf', gamma=gamma, n_components=n_components, random_state=random_state)
Nyst_Sampler = NystInt.fit(Data)
return Nyst_Sampler
del lka, lowrank_feats
# # # # # # # # # # # # # # # #
# Nyström again!
# # # # # # # # # # # # # # # #
print("Running Nystroem Approximation")
factor = np.max(np.linalg.norm(train_features, axis=1))
train_features /= factor
test_features /= factor
dim = min(train_features.shape[1] * 2, train_features.shape[0])
print("Nystroem dim is {}".format(dim))
# Use the Nyström approximation in sklearn
approx = Nystroem(kernel='rbf', gamma=1., n_components=dim)
approx.fit(train_features)
train_features = approx.transform(train_features)
test_features = approx.transform(test_features)
# # # # # # # # # # # # # # # #
# Ridge regression with cross validation
# # # # # # # # # # # # # # # #
style = 'c' if train_features.shape[0] > train_features.shape[1] else 'k'
clf = GridSearchCV(RidgeRegression(), {
'alpha': [(10**i) for i in range(-7, 0)],
'style': [style]
},
n_jobs=4)
clf.fit(train_features, train_onehot)
XtrainT = kcca.transform(ktrain)
XtestT = kcca.transform(ktest)
kccaScores = np.zeros((2,np.alen(nComponents)))
for i,n in enumerate(nComponents):
kccaScores[:,i] = util.classify(XtrainT[:,0:n],XtestT[:,0:n],labelsTrain,labelsTest)
#%% Subsampling methods
kpls = PLSRegression(n_components=150)
nComponents = np.arange(173,2173,100)
# Nystroem method
elapTimeNys = np.zeros(np.shape(nComponents))
kplsScoresNys = np.zeros((2,3))
for i,n in enumerate(nComponents):
nys = Nystroem(n_components=n,gamma=gamma)
nys.fit(Xtrain)
ktrain = nys.transform(Xtrain)
ktest = nys.transform(Xtest)
startTime = timeit.default_timer()
kpls.fit(ktrain,Ytrain)
elapTimeNys[i] = timeit.default_timer() - startTime
XtrainT = kpls.transform(ktrain)
XtestT = kpls.transform(ktest)
if n==573:
kplsScoresNys[:,0] = util.classify(XtrainT,XtestT,labelsTrain,labelsTest)
elif n==1073:
kplsScoresNys[:,1] = util.classify(XtrainT,XtestT,labelsTrain,labelsTest)
elif n==1573:
kplsScoresNys[:,2] = util.classify(XtrainT,XtestT,labelsTrain,labelsTest)
rbf2 = RBFSampler2(gamma=gamma, n_components=nc / 2)
rbf2.fit(Xtr)
# the random state needs to be specified so that the test and train features are the same
clfa2 = svm.LinearSVC(C=C, loss='hinge')
clfa2.fit(rbf2.transform(Xtr), Ytr)
nys = Nystroem(kernel='rbf',
gamma=gamma,
coef0=1,
degree=3,
kernel_params=None,
n_components=nc,
random_state=None)
nys.fit(Xtr)
clf3 = svm.LinearSVC(C=C, loss='hinge')
clf3.fit(nys.transform(Xtr), Ytr)
clf = svm.SVC(kernel='rbf', C=C, gamma=gamma)
clf.fit(Xtr, Ytr)
plt.close('all')
npts = 50
xm, xM = np.min(Xtr), np.max(Xtr)
x = np.linspace(xm, xM, npts)
y = np.linspace(xm, xM, npts)
t = np.array(list(itertools.product(x, y)))
def plotit(z, title):
XtestT = kcca.transform(ktest)
kccaScores = np.zeros((2, np.alen(nComponents)))
for i, n in enumerate(nComponents):
kccaScores[:, i] = util.classify(XtrainT[:, 0:n], XtestT[:, 0:n],
labelsTrain, labelsTest)
#%% Subsampling methods
kpls = PLSRegression(n_components=150)
nComponents = np.arange(173, 2173, 100)
# Nystroem method
elapTimeNys = np.zeros(np.shape(nComponents))
kplsScoresNys = np.zeros((2, 3))
for i, n in enumerate(nComponents):
nys = Nystroem(n_components=n, gamma=gamma)
nys.fit(Xtrain)
ktrain = nys.transform(Xtrain)
ktest = nys.transform(Xtest)
startTime = timeit.default_timer()
kpls.fit(ktrain, Ytrain)
elapTimeNys[i] = timeit.default_timer() - startTime
XtrainT = kpls.transform(ktrain)
XtestT = kpls.transform(ktest)
if n == 573:
kplsScoresNys[:, 0] = util.classify(XtrainT, XtestT, labelsTrain,
labelsTest)
elif n == 1073:
kplsScoresNys[:, 1] = util.classify(XtrainT, XtestT, labelsTrain,
labelsTest)
elif n == 1573:
class NystronemSampler(Transformer):
def __init__(self,
kernel='rbf',
n_components=100,
gamma=1.0,
degree=3,
coef0=1,
random_state=None):
super().__init__("nystronem_sampler", 15, random_state=random_state)
self.input_type = [NUMERICAL, DISCRETE, CATEGORICAL]
self.compound_mode = 'only_new'
self.output_type = NUMERICAL
self.kernel = kernel
self.n_components = n_components
self.gamma = gamma
self.degree = degree
self.coef0 = coef0
self.random_state = random_state
@ease_trans
def operate(self, input_datanode, target_fields=None):
X, y = input_datanode.data
X_new = X[:, target_fields].astype(np.float64)
# Because the pipeline guarantees that each feature is positive,
# clip all values below zero to zero
if self.kernel == 'chi2':
if scipy.sparse.issparse(X_new):
X_new.data[X_new.data < 0] = 0.0
else:
X_new[X_new < 0] = 0.0
if not self.model:
n_components = min(X.shape[0], self.n_components)
self.gamma = float(self.gamma)
self.degree = int(self.degree)
self.coef0 = float(self.coef0)
self.model = Nystroem(kernel=self.kernel,
n_components=n_components,
gamma=self.gamma,
degree=self.degree,
coef0=self.coef0,
random_state=self.random_state)
self.model.fit(X_new.astype(np.float64))
_X = self.model.transform(X_new)
return _X
@staticmethod
def get_hyperparameter_search_space(dataset_properties=None):
if dataset_properties is not None and \
(dataset_properties.get("sparse") is True or
dataset_properties.get("signed") is False):
allow_chi2 = False
else:
allow_chi2 = True
possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine']
if allow_chi2:
possible_kernels.append("chi2")
kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf')
n_components = UniformIntegerHyperparameter("n_components",
10,
2000,
default_value=100,
log=True)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default_value=0.1)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
if allow_chi2:
gamma_kernels.append("chi2")
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_conditions(
[degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def estimateAll(self, loggedData):
numInstances = len(loggedData)
targets = numpy.zeros(numInstances, order='C', dtype=numpy.float64)
null_covariates = scipy.sparse.lil_matrix(
(numInstances, self.numFeatures * self.rankingSize),
dtype=numpy.float64)
target_covariates = scipy.sparse.lil_matrix(
(numInstances, self.numFeatures * self.rankingSize),
dtype=numpy.float64)
print("Starting to create covariates", flush=True)
for j in range(numInstances):
currentDatapoint = loggedData[j]
targets[j] = currentDatapoint[2]
currentQuery = currentDatapoint[0]
currentRanking = currentDatapoint[1]
newRanking = currentDatapoint[3]
nullFeatures = self.loggingPolicy.dataset.features[currentQuery][
currentRanking, :]
nullFeatures.eliminate_zeros()
targetFeatures = self.loggingPolicy.dataset.features[currentQuery][
newRanking, :]
targetFeatures.eliminate_zeros()
null_covariates.data[j] = nullFeatures.data
target_covariates.data[j] = targetFeatures.data
nullIndices = nullFeatures.indices
targetIndices = targetFeatures.indices
for k in range(nullFeatures.shape[0]):
nullIndices[nullFeatures.indptr[k]:nullFeatures.
indptr[k + 1]] += k * self.numFeatures
targetIndices[targetFeatures.indptr[k]:targetFeatures.
indptr[k + 1]] += k * self.numFeatures
null_covariates.rows[j] = nullIndices
target_covariates.rows[j] = targetIndices
if j % 1000 == 0:
print(".", end='', flush=True)
del currentDatapoint
del nullFeatures
del targetFeatures
print("Converting covariates", flush=True)
null_covariates = null_covariates.toarray()
target_covariates = target_covariates.toarray()
scaler = sklearn.preprocessing.MinMaxScaler()
scaler.fit(null_covariates)
s_null_covariates = scaler.transform(null_covariates)
s_target_covariates = scaler.transform(target_covariates)
print("Finished conversion", flush=True)
print("Calculating heuristic kernel param", flush=True)
random_indices = numpy.arange(
0, s_null_covariates.shape[0]) # array of all indices
numpy.random.shuffle(random_indices) # shuffle the array
sample_null_covar = s_null_covariates[
random_indices[:2000]] # get N samples without replacement
sample_target_covar = s_target_covariates[
random_indices[:2000]] # get N samples without replacement
recom_param = (0.5 * self.kernel_param) / numpy.median(
pdist(numpy.vstack([sample_null_covar, sample_target_covar]),
'sqeuclidean'))
print("Computing kernel matrix", flush=True)
m = null_covariates.shape[0]
n = target_covariates.shape[0]
reg_params = self.reg_param / n
if self.approx and m > self.p:
p = self.p
rets = []
for i in range(self.n_approx):
nystroem = Nystroem(gamma=recom_param, n_components=p)
nystroem.fit(s_null_covariates)
nullPhi = nystroem.transform(s_null_covariates)
targetPhi = nystroem.transform(s_target_covariates)
b = numpy.dot(targetPhi.T, numpy.repeat(1.0 / m, m, axis=0))
A = nullPhi.T.dot(nullPhi) + numpy.diag(
numpy.repeat(p * reg_params, p))
beta_vec_approx = nullPhi.dot(
scipy.sparse.linalg.cg(A, b, tol=1e-08, maxiter=5000)[0])
ret = numpy.dot(beta_vec_approx,
targets) / beta_vec_approx.sum()
rets.append(ret)
return numpy.mean(rets)
else:
nullRecomMatrix = self.kernel(s_null_covariates, s_null_covariates,
recom_param)
targetRecomMatrix = self.kernel(s_null_covariates,
s_target_covariates, recom_param)
b = numpy.dot(targetRecomMatrix, numpy.repeat(1.0 / m, m, axis=0))
A = nullRecomMatrix + numpy.diag(numpy.repeat(n * reg_params, n))
print("Finding beta_vec", flush=True)
beta_vec, _ = scipy.sparse.linalg.cg(A, b, tol=1e-08, maxiter=5000)
return numpy.dot(beta_vec, targets) / beta_vec.sum()
class ParametricModelApproximation(object):
"""Approximate a Gaussian Process by a parametric model.
Approximating a Gaussian Process by a parametric model can be useful if
one has to evaluate a sample function from the GP repeatedly or on many
evaluation points as this would become computationally very expensive
with a GP.
Parameters
----------
model : GaussianProcessRegressor
The Gaussian Process which is to be approximated
bounds: list of pair of floats
The boundaries of the data space. This is used when determining the
features of the parametric approximation (they are centered at random
points in the data space)
n_components: int
The number of features/parameters of the parametric model
seed: int
The seed of the random number generator
"""
def __init__(self, model, bounds, n_components, seed):
self.gp = model
self.bounds = bounds
self.n_components = n_components
self.rng = np.random.RandomState(seed)
self.X_space = self.rng.uniform(self.bounds[:, 0], self.bounds[:, 1],
(1000, self.bounds.shape[0]))
assert self.gp.X_fit_.shape[1] == self.X_space.shape[1]
self.kernel = self.gp.kernel_
self.nystr = Nystroem(
n_components=min(self.n_components, self.X_space.shape[0]),
kernel='precomputed', random_state=self.rng)
self.nystr.fit(self.kernel(self.X_space))
def determine_coefs(self, X_query=None, y_query_samples=None, n_samples=1):
""" Determine coefficients of parametric model.
Simulate an evaluation at X_query with outcomes y_query_samples.
Determine coefficients of parametric model the updated GP.
Parameters
----------
X_query : ndarray-like, default: None
The query point at which an additional evaluation is simulated.
If None, a parametric approximation of the unmodified GP is
returned.
y_query_samples: ndarray-like, default: None
The possible outcomes of a query at X_query.
n_samples: int
The number of independent samples of model coefficients from the
Bayesian posterior over model coefficients
"""
if X_query is not None:
X_query = np.asarray(X_query)
X_queried = np.vstack((self.gp.X_fit_, X_query))
else:
X_queried = self.gp.X_fit_
y_queried = self.gp.y_fit_
Phi = self.nystr.transform(self.kernel(self.X_space, X_queried))
A = Phi.T.dot(Phi) + self.gp.alpha * np.eye(Phi.shape[1])
A_inv = np.linalg.inv(A)
cov = self.gp.alpha * A_inv
coefs = \
np.empty((n_samples, self.n_components, y_query_samples.shape[0]))
for i in range(y_query_samples.shape[0]): # XXX: Vectorize
y_queried = np.hstack((self.gp.y_fit_, y_query_samples[i]))
mean = A_inv.dot(Phi.T).dot(y_queried)
coefs[:, :, i] = self.rng.multivariate_normal(mean, cov, n_samples)
return np.array(coefs)
def __call__(self, X, coefs):
""" Evaluate parametric model at X for the given sampled coefficients.
Parameters
----------
X : ndarray-like
The points at which the parametric model is to be evaluated
coefs: ndarray-like
The coefficients of the parametric model.
"""
X = np.atleast_2d(X)
Phi = self.nystr.transform(self.kernel(self.X_space, X))
f = Phi.dot(coefs)
return f
