def test_scaler_without_centering():
rng = np.random.RandomState(42)
X = rng.randn(4, 5)
X[:, 0] = 0.0 # first feature is always of zero
scaler = Scaler(with_mean=False)
X_scaled = scaler.fit(X).transform(X, copy=True)
assert not np.any(np.isnan(X_scaled))
assert_array_almost_equal(
X_scaled.mean(axis=0), [0., -0.01, 2.24, -0.35, -0.78], 2)
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert X_scaled is not X
X_scaled_back = scaler.inverse_transform(X_scaled)
assert X_scaled_back is not X
assert X_scaled_back is not X_scaled
assert_array_almost_equal(X_scaled_back, X)
X_scaled = scale(X, with_mean=False)
assert not np.any(np.isnan(X_scaled))
assert_array_almost_equal(
X_scaled.mean(axis=0), [0., -0.01, 2.24, -0.35, -0.78], 2)
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert X_scaled is not X
X_scaled_back = scaler.inverse_transform(X_scaled)
assert X_scaled_back is not X
assert X_scaled_back is not X_scaled
assert_array_almost_equal(X_scaled_back, X)
def test_scaler_1d():
"""Test scaling of dataset along single axis"""
rng = np.random.RandomState(0)
X = rng.randn(5)
X_orig_copy = X.copy()
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=False)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
# check inverse transform
X_scaled_back = scaler.inverse_transform(X_scaled)
assert_array_almost_equal(X_scaled_back, X_orig_copy)
# Test with 1D list
X = [0., 1., 2, 0.4, 1.]
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=False)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
X_scaled = scale(X)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
def test_scaler_1d():
"""Test scaling of dataset along single axis"""
rng = np.random.RandomState(0)
X = rng.randn(5)
X_orig_copy = X.copy()
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=False)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
# check inverse transform
X_scaled_back = scaler.inverse_transform(X_scaled)
assert_array_almost_equal(X_scaled_back, X_orig_copy)
# Test with 1D list
X = [0., 1., 2, 0.4, 1.]
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=False)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
X_scaled = scale(X)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
def test_scaler_without_centering():
rng = np.random.RandomState(42)
X = rng.randn(4, 5)
X[:, 0] = 0.0 # first feature is always of zero
X_csr = sp.csr_matrix(X)
scaler = Scaler(with_mean=False).fit(X)
X_scaled = scaler.transform(X, copy=True)
assert_false(np.any(np.isnan(X_scaled)))
scaler_csr = Scaler(with_mean=False).fit(X_csr)
X_csr_scaled = scaler_csr.transform(X_csr, copy=True)
assert_false(np.any(np.isnan(X_csr_scaled.data)))
assert_equal(scaler.mean_, scaler_csr.mean_)
assert_array_almost_equal(scaler.std_, scaler_csr.std_)
assert_array_almost_equal(
X_scaled.mean(axis=0), [0., -0.01, 2.24, -0.35, -0.78], 2)
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
X_csr_scaled_mean, X_csr_scaled_std = mean_variance_axis0(X_csr_scaled)
assert_array_almost_equal(X_csr_scaled_mean, X_scaled.mean(axis=0))
assert_array_almost_equal(X_csr_scaled_std, X_scaled.std(axis=0))
# Check that X has not been modified (copy)
assert_true(X_scaled is not X)
assert_true(X_csr_scaled is not X_csr)
X_scaled_back = scaler.inverse_transform(X_scaled)
assert_true(X_scaled_back is not X)
assert_true(X_scaled_back is not X_scaled)
assert_array_almost_equal(X_scaled_back, X)
X_csr_scaled_back = scaler_csr.inverse_transform(X_csr_scaled)
assert_true(X_csr_scaled_back is not X_csr)
assert_true(X_csr_scaled_back is not X_csr_scaled)
assert_array_almost_equal(X_scaled_back, X)
def test_scaler_without_centering():
rng = np.random.RandomState(42)
X = rng.randn(4, 5)
X[:, 0] = 0.0 # first feature is always of zero
X_csr = sp.csr_matrix(X)
scaler = Scaler(with_mean=False).fit(X)
X_scaled = scaler.transform(X, copy=True)
assert_false(np.any(np.isnan(X_scaled)))
scaler_csr = Scaler(with_mean=False).fit(X_csr)
X_csr_scaled = scaler_csr.transform(X_csr, copy=True)
assert_false(np.any(np.isnan(X_csr_scaled.data)))
assert_equal(scaler.mean_, scaler_csr.mean_)
assert_array_almost_equal(scaler.std_, scaler_csr.std_)
assert_array_almost_equal(X_scaled.mean(axis=0),
[0., -0.01, 2.24, -0.35, -0.78], 2)
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
X_csr_scaled_mean, X_csr_scaled_std = mean_variance_axis0(X_csr_scaled)
assert_array_almost_equal(X_csr_scaled_mean, X_scaled.mean(axis=0))
assert_array_almost_equal(X_csr_scaled_std, X_scaled.std(axis=0))
# Check that X has not been modified (copy)
assert_true(X_scaled is not X)
assert_true(X_csr_scaled is not X_csr)
X_scaled_back = scaler.inverse_transform(X_scaled)
assert_true(X_scaled_back is not X)
assert_true(X_scaled_back is not X_scaled)
assert_array_almost_equal(X_scaled_back, X)
X_csr_scaled_back = scaler_csr.inverse_transform(X_csr_scaled)
assert_true(X_csr_scaled_back is not X_csr)
assert_true(X_csr_scaled_back is not X_csr_scaled)
assert_array_almost_equal(X_scaled_back, X)
def test_scaler_2d_arrays():
"""Test scaling of 2d array along first axis"""
rng = np.random.RandomState(0)
X = rng.randn(4, 5)
X[:, 0] = 0.0 # first feature is always of zero
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=True)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert_true(X_scaled is not X)
# check inverse transform
X_scaled_back = scaler.inverse_transform(X_scaled)
assert_true(X_scaled_back is not X)
assert_true(X_scaled_back is not X_scaled)
assert_array_almost_equal(X_scaled_back, X)
X_scaled = scale(X, axis=1, with_std=False)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0])
X_scaled = scale(X, axis=1, with_std=True)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=1), 4 * [1.0])
# Check that the data hasn't been modified
assert_true(X_scaled is not X)
X_scaled = scaler.fit(X).transform(X, copy=False)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert_true(X_scaled is X)
X = rng.randn(4, 5)
X[:, 0] = 1.0 # first feature is a constant, non zero feature
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=True)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert_true(X_scaled is not X)
def test_scaler_2d_arrays():
"""Test scaling of 2d array along first axis"""
rng = np.random.RandomState(0)
X = rng.randn(4, 5)
X[:, 0] = 0.0 # first feature is always of zero
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=True)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert_true(X_scaled is not X)
# check inverse transform
X_scaled_back = scaler.inverse_transform(X_scaled)
assert_true(X_scaled_back is not X)
assert_true(X_scaled_back is not X_scaled)
assert_array_almost_equal(X_scaled_back, X)
X_scaled = scale(X, axis=1, with_std=False)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0])
X_scaled = scale(X, axis=1, with_std=True)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=1), 4 * [1.0])
# Check that the data hasn't been modified
assert_true(X_scaled is not X)
X_scaled = scaler.fit(X).transform(X, copy=False)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert_true(X_scaled is X)
X = rng.randn(4, 5)
X[:, 0] = 1.0 # first feature is a constant, non zero feature
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=True)
assert_false(np.any(np.isnan(X_scaled)))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert_true(X_scaled is not X)
def test_scaler():
"""Test scaling of dataset along all axis"""
# First test with 1D data
X = np.random.randn(5)
X_orig_copy = X.copy()
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=False)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
# check inverse transform
X_scaled_back = scaler.inverse_transform(X_scaled)
assert_array_almost_equal(X_scaled_back, X_orig_copy)
# Test with 1D list
X = [0., 1., 2, 0.4, 1.]
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=False)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
X_scaled = scale(X)
assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
# Test with 2D data
X = np.random.randn(4, 5)
X[:, 0] = 0.0 # first feature is always of zero
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=True)
assert not np.any(np.isnan(X_scaled))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert X_scaled is not X
# check inverse transform
X_scaled_back = scaler.inverse_transform(X_scaled)
assert X_scaled_back is not X
assert X_scaled_back is not X_scaled
assert_array_almost_equal(X_scaled_back, X)
X_scaled = scale(X, axis=1, with_std=False)
assert not np.any(np.isnan(X_scaled))
assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0])
X_scaled = scale(X, axis=1, with_std=True)
assert not np.any(np.isnan(X_scaled))
assert_array_almost_equal(X_scaled.mean(axis=1), 4 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=1), 4 * [1.0])
# Check that the data hasn't been modified
assert X_scaled is not X
X_scaled = scaler.fit(X).transform(X, copy=False)
assert not np.any(np.isnan(X_scaled))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert X_scaled is X
X = np.random.randn(4, 5)
X[:, 0] = 1.0 # first feature is a constant, non zero feature
scaler = Scaler()
X_scaled = scaler.fit(X).transform(X, copy=True)
assert not np.any(np.isnan(X_scaled))
assert_array_almost_equal(X_scaled.mean(axis=0), 5 * [0.0])
assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
# Check that X has not been copied
assert X_scaled is not X
class KMPBase(BaseEstimator):
def __init__(self,
n_nonzero_coefs=0.3,
loss=None,
# components (basis functions)
init_components=None,
n_components=None,
check_duplicates=False,
scale=False,
scale_y=False,
# back-fitting
n_refit=5,
estimator=None,
# metric
metric="linear", gamma=0.1, coef0=1, degree=4,
# validation
X_val=None, y_val=None,
n_validate=1,
epsilon=0,
score_func=None,
# misc
random_state=None, verbose=0, n_jobs=1):
if n_nonzero_coefs < 0:
raise AttributeError("n_nonzero_coefs should be > 0.")
self.n_nonzero_coefs = n_nonzero_coefs
self.loss = loss
self.init_components = init_components
self.n_components = n_components
self.check_duplicates = check_duplicates
self.scale = scale
self.scale_y = scale_y
self.n_refit = n_refit
self.estimator = estimator
self.metric = metric
self.gamma = gamma
self.coef0 = coef0
self.degree = degree
self.X_val = X_val
self.y_val = y_val
self.n_validate = n_validate
self.epsilon = epsilon
self.score_func = score_func
self.random_state = random_state
self.verbose = verbose
self.n_jobs = n_jobs
def _kernel_params(self):
return {"gamma" : self.gamma,
"degree" : self.degree,
"coef0" : self.coef0}
def _get_estimator(self):
if self.estimator is None:
estimator = LinearRegression()
else:
estimator = clone(self.estimator)
estimator.fit_intercept = False
return estimator
def _get_loss(self):
if self.loss == "squared":
return SquaredLoss()
else:
return None
def _pre_fit(self, X, y):
random_state = check_random_state(self.random_state)
if self.scale_y:
self.y_scaler_ = Scaler(copy=True).fit(y)
y = self.y_scaler_.transform(y)
if self.metric == "precomputed":
self.components_ = None
n_components = X.shape[1]
else:
if self.init_components is None:
if self.verbose: print "Selecting components..."
self.components_ = select_components(X, y,
self.n_components,
random_state=random_state)
else:
self.components_ = self.init_components
n_components = self.components_.shape[0]
n_nonzero_coefs = self.n_nonzero_coefs
if 0 < n_nonzero_coefs and n_nonzero_coefs <= 1:
n_nonzero_coefs = int(n_nonzero_coefs * n_components)
n_nonzero_coefs = int(n_nonzero_coefs)
if n_nonzero_coefs > n_components:
raise AttributeError("n_nonzero_coefs cannot be bigger than "
"n_components.")
if self.verbose: print "Computing dictionary..."
start = time.time()
K = pairwise_kernels(X, self.components_, metric=self.metric,
filter_params=True, n_jobs=self.n_jobs,
**self._kernel_params())
if self.verbose: print "Done in", time.time() - start, "seconds"
if self.scale:
if self.verbose: print "Scaling dictionary"
start = time.time()
copy = True if self.metric == "precomputed" else False
self.scaler_ = Scaler(copy=copy)
K = self.scaler_.fit_transform(K)
if self.verbose: print "Done in", time.time() - start, "seconds"
# FIXME: this allocates a lot of intermediary memory
norms = np.sqrt(np.sum(K ** 2, axis=0))
return n_nonzero_coefs, K, y, norms
def _fit_multi(self, K, y, Y, n_nonzero_coefs, norms):
if self.verbose: print "Starting training..."
start = time.time()
coef = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
delayed(_run_iterator)(self._get_estimator(),
self._get_loss(),
K, Y[:, i], n_nonzero_coefs, norms,
self.n_refit, self.check_duplicates)
for i in xrange(Y.shape[1]))
self.coef_ = np.array(coef)
if self.verbose: print "Done in", time.time() - start, "seconds"
def _score(self, y_true, y_pred):
if self.score_func == "auc":
return auc(y_true, y_pred)
if hasattr(self, "lb_"):
y_pred = self.lb_.inverse_transform(y_pred, threshold=0.5)
if self.score_func is None:
return np.mean(y_true == y_pred)
else:
return self.score_func(y_true, y_pred)
else:
# FIXME: no need to ravel y_pred if y_true is 2d!
return -np.mean((y_true - y_pred.ravel()) ** 2)
def _fit_multi_with_validation(self, K, y, Y, n_nonzero_coefs, norms):
iterators = [FitIterator(self._get_estimator(), self._get_loss(),
K, Y[:, i], n_nonzero_coefs, norms,
self.n_refit, self.check_duplicates,
self.verbose)
for i in xrange(Y.shape[1])]
if self.verbose: print "Computing validation dictionary..."
start = time.time()
K_val = pairwise_kernels(self.X_val, self.components_,
metric=self.metric,
filter_params=True,
n_jobs=self.n_jobs,
**self._kernel_params())
if self.verbose: print "Done in", time.time() - start, "seconds"
if self.scale:
K_val = self.scaler_.transform(K_val)
y_val = self.y_val
if self.scale_y:
y_val = self.y_scaler_.transform(y_val)
if self.verbose: print "Starting training..."
start = time.time()
best_score = -np.inf
validation_scores = []
training_scores = []
iterations = []
for i in xrange(1, n_nonzero_coefs + 1):
iterators = [it.next() for it in iterators]
#iterators = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
#delayed(_run_iterator)(it) for it in iterators)
coef = np.array([it.coef_ for it in iterators])
y_train_pred = np.array([it.y_train_ for it in iterators]).T
if i % self.n_validate == 0:
if self.verbose >= 2:
print "Validating %d/%d..." % (i, n_nonzero_coefs)
y_val_pred = np.dot(K_val, coef.T)
validation_score = self._score(y_val, y_val_pred)
training_score = self._score(y, y_train_pred)
if validation_score > best_score:
self.coef_ = coef.copy()
best_score = np.abs(validation_score)
validation_scores.append(np.abs(validation_score))
training_scores.append(np.abs(training_score))
iterations.append(i)
if len(iterations) > 2 and self.epsilon > 0:
diff = (validation_scores[-1] - validation_scores[-2])
diff /= validation_scores[0]
if abs(diff) < self.epsilon:
if self.verbose:
print "Converged at iteration", i
break
self.validation_scores_ = np.array(validation_scores)
self.training_scores_ = np.array(training_scores)
self.iterations_ = np.array(iterations)
self.best_score_ = best_score
if self.verbose: print "Done in", time.time() - start, "seconds"
def _fit(self, K, y, Y, n_nonzero_coefs, norms):
if self.X_val is not None and self.y_val is not None:
meth = self._fit_multi_with_validation
else:
meth = self._fit_multi
meth(K, y, Y, n_nonzero_coefs, norms)
def _post_fit(self):
if self.metric != "precomputed":
used_basis = np.sum(self.coef_ != 0, axis=0, dtype=bool)
self.coef_ = self.coef_[:, used_basis]
self.components_ = self.components_[used_basis]
def decision_function(self, X):
K = pairwise_kernels(X, self.components_, metric=self.metric,
filter_params=True, n_jobs=self.n_jobs,
**self._kernel_params())
if self.scale:
K = self.scaler_.transform(K)
pred = np.dot(K, self.coef_.T)
if self.scale_y:
pred = self.y_scaler_.inverse_transform(pred)
return pred
Xtrain = Xtrain[~np.isnan(np.nansum(Xtrain, axis=1)), :]
if np.unique(ytrain).shape[0] > 1:
# feature selection (find the 50% most discriminative channels)
fs.fit(Xtrain, ytrain) # find
Xtrain = fs.transform(Xtrain) # remove unnecessary channels
# normalization
scaler.fit(Xtrain) # find
Xtrain = scaler.transform(Xtrain) # apply zscore
# SVM fit
clf.fit(Xtrain, ytrain, sample_weight=sw_train)
# retrieve hyperplan feature identification
coef[split, fold, d, :, :] = 0 # initialize
# --- univariate
uni_features = fs.pvalues_ <= stats.scoreatpercentile(fs.pvalues_, fs.percentile)
# --- multivariate
coef[split, fold, d, :, uni_features] = scaler.inverse_transform(clf.coef_).T
# predict cross val (deal with NaN in testing)
# generalize across all time points
for d_tg in range(0, n_dims_tg):
sys.stdout.write("*")
sys.stdout.flush()
# select data
Xtest = Xm_shfl[test, :, dims_tg[d, d_tg]]
# handles NaNs
test_nan = np.isnan(np.nansum(Xtest, axis=1))
Xtest = Xtest[~test_nan, :]
# preproc
Xtest = fs.transform(Xtest)
Xtest = scaler.transform(Xtest)
# predict
if (Xtest.shape[0] - np.sum(test_nan)) > 0:
Xtrain = Xtrain[~np.isnan(np.nansum(Xtrain, axis=1)), :]
if np.unique(ytrain).shape[0] > 1:
# feature selection (find the 50% most discriminative channels)
fs.fit(Xtrain, ytrain) # find
Xtrain = fs.transform(Xtrain) # remove unnecessary channels
# normalization
scaler.fit(Xtrain) # find
Xtrain = scaler.transform(Xtrain) # apply zscore
# SVM fit
clf.fit(Xtrain, ytrain, sample_weight=sw_train)
# retrieve hyperplan feature identification
coef[split, fold, d, :, :] = 0 # initialize
#--- univariate
uni_features = fs.pvalues_ <= stats.scoreatpercentile(fs.pvalues_, fs.percentile)
#--- multivariate
coef[split, fold, d, :, uni_features] = scaler.inverse_transform(clf.coef_).T
# predict cross val (deal with NaN in testing)
# generalize across all time points
for d_tg in range(0, n_dims_tg):
sys.stdout.write("*")
sys.stdout.flush()
# select data
Xtest = Xm_shfl[test, :, dims_tg[d, d_tg]]
# handles NaNs
test_nan = np.isnan(np.nansum(Xtest, axis=1))
Xtest = Xtest[~test_nan, :]
# preproc
Xtest = fs.transform(Xtest)
Xtest = scaler.transform(Xtest)
# predict
if (Xtest.shape[0] - np.sum(test_nan)) > 0: