def test_dtype_match():
rng = np.random.RandomState(0)
alpha = 1.0
n_samples, n_features = 6, 5
X_64 = rng.randn(n_samples, n_features)
y_64 = rng.randn(n_samples)
X_32 = X_64.astype(np.float32)
y_32 = y_64.astype(np.float32)
solvers = ["svd", "sparse_cg", "cholesky", "lsqr"]
for solver in solvers:
# Check type consistency 32bits
ridge_32 = Ridge(alpha=alpha, solver=solver)
ridge_32.fit(X_32, y_32)
coef_32 = ridge_32.coef_
# Check type consistency 64 bits
ridge_64 = Ridge(alpha=alpha, solver=solver)
ridge_64.fit(X_64, y_64)
coef_64 = ridge_64.coef_
# Do the actual checks at once for easier debug
assert coef_32.dtype == X_32.dtype
assert coef_64.dtype == X_64.dtype
assert ridge_32.predict(X_32).dtype == X_32.dtype
assert ridge_64.predict(X_64).dtype == X_64.dtype
assert_almost_equal(ridge_32.coef_, ridge_64.coef_, decimal=5)
def test_dtype_match(solver):
rng = np.random.RandomState(0)
alpha = 1.0
n_samples, n_features = 6, 5
X_64 = rng.randn(n_samples, n_features)
y_64 = rng.randn(n_samples)
X_32 = X_64.astype(np.float32)
y_32 = y_64.astype(np.float32)
# Check type consistency 32bits
ridge_32 = Ridge(alpha=alpha, solver=solver, max_iter=500, tol=1e-10,)
ridge_32.fit(X_32, y_32)
coef_32 = ridge_32.coef_
# Check type consistency 64 bits
ridge_64 = Ridge(alpha=alpha, solver=solver, max_iter=500, tol=1e-10,)
ridge_64.fit(X_64, y_64)
coef_64 = ridge_64.coef_
# Do the actual checks at once for easier debug
assert coef_32.dtype == X_32.dtype
assert coef_64.dtype == X_64.dtype
assert ridge_32.predict(X_32).dtype == X_32.dtype
assert ridge_64.predict(X_64).dtype == X_64.dtype
assert_allclose(ridge_32.coef_, ridge_64.coef_, rtol=1e-4)
def test_dtype_match_cholesky():
# Test different alphas in cholesky solver to ensure full coverage.
# This test is separated from test_dtype_match for clarity.
rng = np.random.RandomState(0)
alpha = (1.0, 0.5)
n_samples, n_features, n_target = 6, 7, 2
X_64 = rng.randn(n_samples, n_features)
y_64 = rng.randn(n_samples, n_target)
X_32 = X_64.astype(np.float32)
y_32 = y_64.astype(np.float32)
# Check type consistency 32bits
ridge_32 = Ridge(alpha=alpha, solver='cholesky')
ridge_32.fit(X_32, y_32)
coef_32 = ridge_32.coef_
# Check type consistency 64 bits
ridge_64 = Ridge(alpha=alpha, solver='cholesky')
ridge_64.fit(X_64, y_64)
coef_64 = ridge_64.coef_
# Do all the checks at once, like this is easier to debug
assert coef_32.dtype == X_32.dtype
assert coef_64.dtype == X_64.dtype
assert ridge_32.predict(X_32).dtype == X_32.dtype
assert ridge_64.predict(X_64).dtype == X_64.dtype
assert_almost_equal(ridge_32.coef_, ridge_64.coef_, decimal=5)
def test_sparse_design_with_sample_weights():
# Sample weights must work with sparse matrices
n_sampless = [2, 3]
n_featuress = [3, 2]
rng = np.random.RandomState(42)
sparse_matrix_converters = [sp.coo_matrix,
sp.csr_matrix,
sp.csc_matrix,
sp.lil_matrix,
sp.dok_matrix
]
sparse_ridge = Ridge(alpha=1., fit_intercept=False)
dense_ridge = Ridge(alpha=1., fit_intercept=False)
for n_samples, n_features in zip(n_sampless, n_featuress):
X = rng.randn(n_samples, n_features)
y = rng.randn(n_samples)
sample_weights = rng.randn(n_samples) ** 2 + 1
for sparse_converter in sparse_matrix_converters:
X_sparse = sparse_converter(X)
sparse_ridge.fit(X_sparse, y, sample_weight=sample_weights)
dense_ridge.fit(X, y, sample_weight=sample_weights)
assert_array_almost_equal(sparse_ridge.coef_, dense_ridge.coef_,
decimal=6)
def test_raises_value_error_if_sample_weights_greater_than_1d():
# Sample weights must be either scalar or 1D
n_sampless = [2, 3]
n_featuress = [3, 2]
rng = np.random.RandomState(42)
for n_samples, n_features in zip(n_sampless, n_featuress):
X = rng.randn(n_samples, n_features)
y = rng.randn(n_samples)
sample_weights_OK = rng.randn(n_samples) ** 2 + 1
sample_weights_OK_1 = 1.0
sample_weights_OK_2 = 2.0
sample_weights_not_OK = sample_weights_OK[:, np.newaxis]
sample_weights_not_OK_2 = sample_weights_OK[np.newaxis, :]
ridge = Ridge(alpha=1)
# make sure the "OK" sample weights actually work
ridge.fit(X, y, sample_weights_OK)
ridge.fit(X, y, sample_weights_OK_1)
ridge.fit(X, y, sample_weights_OK_2)
def fit_ridge_not_ok():
ridge.fit(X, y, sample_weights_not_OK)
def fit_ridge_not_ok_2():
ridge.fit(X, y, sample_weights_not_OK_2)
assert_raise_message(ValueError, "Sample weights must be 1D array or scalar", fit_ridge_not_ok)
assert_raise_message(ValueError, "Sample weights must be 1D array or scalar", fit_ridge_not_ok_2)
def _test_ridge_loo(filter_):
# test that can work with both dense or sparse matrices
n_samples = X_diabetes.shape[0]
ret = []
ridge_gcv = _RidgeGCV(fit_intercept=False)
ridge = Ridge(fit_intercept=False)
# generalized cross-validation (efficient leave-one-out)
K, v, Q = ridge_gcv._pre_compute(X_diabetes, y_diabetes)
errors, c = ridge_gcv._errors(v, Q, y_diabetes, 1.0)
values, c = ridge_gcv._values(K, v, Q, y_diabetes, 1.0)
# brute-force leave-one-out: remove one example at a time
errors2 = []
values2 = []
for i in range(n_samples):
sel = np.arange(n_samples) != i
X_new = X_diabetes[sel]
y_new = y_diabetes[sel]
ridge.fit(X_new, y_new)
value = ridge.predict([X_diabetes[i]])[0]
error = (y_diabetes[i] - value) ** 2
errors2.append(error)
values2.append(value)
# check that efficient and brute-force LOO give same results
assert_almost_equal(errors, errors2)
assert_almost_equal(values, values2)
# check best alpha
ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
best_alpha = ridge_gcv.best_alpha
ret.append(best_alpha)
# check that we get same best alpha with custom loss_func
ridge_gcv2 = _RidgeGCV(fit_intercept=False, loss_func=mean_squared_error)
ridge_gcv2.fit(filter_(X_diabetes), y_diabetes)
assert_equal(ridge_gcv2.best_alpha, best_alpha)
# check that we get same best alpha with sample weights
ridge_gcv.fit(filter_(X_diabetes), y_diabetes,
sample_weight=np.ones(n_samples))
assert_equal(ridge_gcv.best_alpha, best_alpha)
# simulate several responses
Y = np.vstack((y_diabetes, y_diabetes)).T
ridge_gcv.fit(filter_(X_diabetes), Y)
Y_pred = ridge_gcv.predict(filter_(X_diabetes))
ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
y_pred = ridge_gcv.predict(filter_(X_diabetes))
assert_array_almost_equal(np.vstack((y_pred, y_pred)).T,
Y_pred, decimal=5)
return ret
def _test_tolerance(filter_):
ridge = Ridge(tol=1e-5, fit_intercept=False)
ridge.fit(filter_(X_diabetes), y_diabetes)
score = ridge.score(filter_(X_diabetes), y_diabetes)
ridge2 = Ridge(tol=1e-3, fit_intercept=False)
ridge2.fit(filter_(X_diabetes), y_diabetes)
score2 = ridge2.score(filter_(X_diabetes), y_diabetes)
assert score >= score2
def _test_tolerance(filter_):
ridge = Ridge(tol=1e-5)
ridge.fit(filter_(X_diabetes), y_diabetes)
score = ridge.score(filter_(X_diabetes), y_diabetes)
ridge2 = Ridge(tol=1e-3)
ridge2.fit(filter_(X_diabetes), y_diabetes)
score2 = ridge2.score(filter_(X_diabetes), y_diabetes)
assert_true(score >= score2)
def test_ridge_singular():
# test on a singular matrix
rng = np.random.RandomState(0)
n_samples, n_features = 6, 6
y = rng.randn(n_samples // 2)
y = np.concatenate((y, y))
X = rng.randn(n_samples // 2, n_features)
X = np.concatenate((X, X), axis=0)
ridge = Ridge(alpha=0)
ridge.fit(X, y)
assert_greater(ridge.score(X, y), 0.9)
def _test_multi_ridge_diabetes(filter_):
# simulate several responses
Y = np.vstack((y_diabetes, y_diabetes)).T
n_features = X_diabetes.shape[1]
ridge = Ridge(fit_intercept=False)
ridge.fit(filter_(X_diabetes), Y)
assert_equal(ridge.coef_.shape, (2, n_features))
Y_pred = ridge.predict(filter_(X_diabetes))
ridge.fit(filter_(X_diabetes), y_diabetes)
y_pred = ridge.predict(filter_(X_diabetes))
assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=3)
def test_ridge():
# Ridge regression convergence test using score
# TODO: for this test to be robust, we should use a dataset instead
# of np.random.
rng = np.random.RandomState(0)
alpha = 1.0
for solver in ("svd", "sparse_cg", "cholesky", "lsqr"):
# With more samples than features
n_samples, n_features = 6, 5
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
ridge = Ridge(alpha=alpha, solver=solver)
ridge.fit(X, y)
assert_equal(ridge.coef_.shape, (X.shape[1], ))
assert_greater(ridge.score(X, y), 0.47)
if solver == "cholesky":
# Currently the only solver to support sample_weight.
ridge.fit(X, y, sample_weight=np.ones(n_samples))
assert_greater(ridge.score(X, y), 0.47)
# With more features than samples
n_samples, n_features = 5, 10
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
ridge = Ridge(alpha=alpha, solver=solver)
ridge.fit(X, y)
assert_greater(ridge.score(X, y), .9)
if solver == "cholesky":
# Currently the only solver to support sample_weight.
ridge.fit(X, y, sample_weight=np.ones(n_samples))
assert_greater(ridge.score(X, y), 0.9)
def test_ridge():
"""Ridge regression convergence test using score
TODO: for this test to be robust, we should use a dataset instead
of np.random.
"""
alpha = 1.0
# With more samples than features
n_samples, n_features = 6, 5
y = np.random.randn(n_samples)
X = np.random.randn(n_samples, n_features)
ridge = Ridge(alpha=alpha)
ridge.fit(X, y)
assert_equal(ridge.coef_.shape, (X.shape[1],))
assert ridge.score(X, y) > 0.5
ridge.fit(X, y, sample_weight=np.ones(n_samples))
assert ridge.score(X, y) > 0.5
# With more features than samples
n_samples, n_features = 5, 10
y = np.random.randn(n_samples)
X = np.random.randn(n_samples, n_features)
ridge = Ridge(alpha=alpha)
ridge.fit(X, y)
assert ridge.score(X, y) > 0.9
ridge.fit(X, y, sample_weight=np.ones(n_samples))
assert ridge.score(X, y) > 0.9
def test_ridge():
"""Ridge regression convergence test using score
TODO: for this test to be robust, we should use a dataset instead
of np.random.
"""
alpha = 1.0
for solver in ("sparse_cg", "dense_cholesky", "lsqr"):
# With more samples than features
n_samples, n_features = 6, 5
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
ridge = Ridge(alpha=alpha, solver=solver)
ridge.fit(X, y)
assert_equal(ridge.coef_.shape, (X.shape[1], ))
assert_greater(ridge.score(X, y), 0.47)
ridge.fit(X, y, sample_weight=np.ones(n_samples))
assert_greater(ridge.score(X, y), 0.47)
# With more features than samples
n_samples, n_features = 5, 10
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
ridge = Ridge(alpha=alpha, solver=solver)
ridge.fit(X, y)
assert_greater(ridge.score(X, y), .9)
ridge.fit(X, y, sample_weight=np.ones(n_samples))
assert_greater(ridge.score(X, y), 0.9)
def test_fit_simple_backupsklearn():
df = pd.read_csv("./open_data/simple.txt", delim_whitespace=True)
X = np.array(df.iloc[:, :df.shape[1] - 1], dtype='float32', order='C')
y = np.array(df.iloc[:, df.shape[1] - 1], dtype='float32', order='C')
Solver = h2o4gpu.Ridge
enet = Solver(glm_stop_early=False)
print("h2o4gpu fit()")
enet.fit(X, y)
print("h2o4gpu predict()")
print(enet.predict(X))
print("h2o4gpu score()")
print(enet.score(X,y))
enet_wrapper = Solver(normalize=True, random_state=1234)
print("h2o4gpu scikit wrapper fit()")
enet_wrapper.fit(X, y)
print("h2o4gpu scikit wrapper predict()")
print(enet_wrapper.predict(X))
print("h2o4gpu scikit wrapper score()")
print(enet_wrapper.score(X, y))
from sklearn.linear_model.ridge import Ridge
enet_sk = Ridge(normalize=True, random_state=1234)
print("Scikit fit()")
enet_sk.fit(X, y)
print("Scikit predict()")
print(enet_sk.predict(X))
print("Scikit score()")
print(enet_sk.score(X, y))
enet_sk_coef = csr_matrix(enet_sk.coef_, dtype=np.float32).toarray()
print(enet_sk.coef_)
print(enet_sk_coef)
print(enet_wrapper.coef_)
print(enet_sk.intercept_)
print(enet_wrapper.intercept_)
print(enet_sk.n_iter_)
print(enet_wrapper.n_iter_)
print("Coeffs, intercept, and n_iters should match")
assert np.allclose(enet_wrapper.coef_, enet_sk_coef)
assert np.allclose(enet_wrapper.intercept_, enet_sk.intercept_)
def test_ridge_sample_weights():
rng = np.random.RandomState(0)
for solver in ("cholesky", ):
for n_samples, n_features in ((6, 5), (5, 10)):
for alpha in (1.0, 1e-2):
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
sample_weight = 1 + rng.rand(n_samples)
coefs = ridge_regression(X, y,
alpha=alpha,
sample_weight=sample_weight,
solver=solver)
# Sample weight can be implemented via a simple rescaling
# for the square loss.
coefs2 = ridge_regression(
X * np.sqrt(sample_weight)[:, np.newaxis],
y * np.sqrt(sample_weight),
alpha=alpha, solver=solver)
assert_array_almost_equal(coefs, coefs2)
# Test for fit_intercept = True
est = Ridge(alpha=alpha, solver=solver)
est.fit(X, y, sample_weight=sample_weight)
# Check using Newton's Method
# Quadratic function should be solved in a single step.
# Initialize
sample_weight = np.sqrt(sample_weight)
X_weighted = sample_weight[:, np.newaxis] * (
np.column_stack((np.ones(n_samples), X)))
y_weighted = y * sample_weight
# Gradient is (X*coef-y)*X + alpha*coef_[1:]
# Remove coef since it is initialized to zero.
grad = -np.dot(y_weighted, X_weighted)
# Hessian is (X.T*X) + alpha*I except that the first
# diagonal element should be zero, since there is no
# penalization of intercept.
diag = alpha * np.ones(n_features + 1)
diag[0] = 0.
hess = np.dot(X_weighted.T, X_weighted)
hess.flat[::n_features + 2] += diag
coef_ = - np.dot(linalg.inv(hess), grad)
assert_almost_equal(coef_[0], est.intercept_)
assert_array_almost_equal(coef_[1:], est.coef_)
def test_ridge_intercept():
# Test intercept with multiple targets GH issue #708
rng = np.random.RandomState(0)
n_samples, n_features = 5, 10
X = rng.randn(n_samples, n_features)
y = rng.randn(n_samples)
Y = np.c_[y, 1. + y]
ridge = Ridge()
ridge.fit(X, y)
intercept = ridge.intercept_
ridge.fit(X, Y)
assert_almost_equal(ridge.intercept_[0], intercept)
assert_almost_equal(ridge.intercept_[1], intercept + 1.)
def test_n_iter():
# Test that self.n_iter_ is correct.
n_targets = 2
X, y = X_diabetes, y_diabetes
y_n = np.tile(y, (n_targets, 1)).T
for max_iter in range(1, 4):
for solver in ('sag', 'saga', 'lsqr'):
reg = Ridge(solver=solver, max_iter=max_iter, tol=1e-12)
reg.fit(X, y_n)
assert_array_equal(reg.n_iter_, np.tile(max_iter, n_targets))
for solver in ('sparse_cg', 'svd', 'cholesky'):
reg = Ridge(solver=solver, max_iter=1, tol=1e-1)
reg.fit(X, y_n)
assert_equal(reg.n_iter_, None)
def BasisPrediction(degree, function):
# Basis function Prediction
res_pred = []
res_mse = []
for d in degree:
#title = 'KRSS, ' + str(typ) + ", degree: " + str(d) + ", lambda: 0.1"
K = kernel_ridge_Basis(train_x, function, d)
K_test = kernel_ridge_Basis(test_x, function, d)
clf = Ridge()
clf.fit(K, train_y)
pred_y = clf.predict(K_test)
res_pred.append(pred_y)
mse = compute_MSE(test_y, pred_y)
print('mse: ', mse)
res_mse.append(mse)
return res_pred, res_mse
def test_n_iter():
# Test that self.n_iter_ is correct.
n_targets = 2
X, y = X_diabetes, y_diabetes
y_n = np.tile(y, (n_targets, 1)).T
for max_iter in range(1, 4):
for solver in ('sag', 'saga', 'lsqr'):
reg = Ridge(solver=solver, max_iter=max_iter, tol=1e-12)
reg.fit(X, y_n)
assert_array_equal(reg.n_iter_, np.tile(max_iter, n_targets))
for solver in ('sparse_cg', 'svd', 'cholesky'):
reg = Ridge(solver=solver, max_iter=1, tol=1e-1)
reg.fit(X, y_n)
assert_equal(reg.n_iter_, None)
def test_ridge_intercept():
# Test intercept with multiple targets GH issue #708
rng = np.random.RandomState(0)
n_samples, n_features = 5, 10
X = rng.randn(n_samples, n_features)
y = rng.randn(n_samples)
Y = np.c_[y, 1. + y]
ridge = Ridge()
ridge.fit(X, y)
intercept = ridge.intercept_
ridge.fit(X, Y)
assert_almost_equal(ridge.intercept_[0], intercept)
assert_almost_equal(ridge.intercept_[1], intercept + 1.)
def test_ridge_fit_intercept_sparse():
X, y = make_regression(n_samples=1000, n_features=2, n_informative=2, bias=10.0, random_state=42)
X_csr = sp.csr_matrix(X)
dense = Ridge(alpha=1.0, tol=1.0e-15, solver="sag", fit_intercept=True)
sparse = Ridge(alpha=1.0, tol=1.0e-15, solver="sag", fit_intercept=True)
dense.fit(X, y)
sparse.fit(X_csr, y)
assert_almost_equal(dense.intercept_, sparse.intercept_)
assert_array_almost_equal(dense.coef_, sparse.coef_)
# test the solver switch and the corresponding warning
sparse = Ridge(alpha=1.0, tol=1.0e-15, solver="lsqr", fit_intercept=True)
assert_warns(UserWarning, sparse.fit, X_csr, y)
assert_almost_equal(dense.intercept_, sparse.intercept_)
assert_array_almost_equal(dense.coef_, sparse.coef_)
def test_ridge_sample_weights():
# TODO: loop over sparse data as well
# Note: parametrizing this test with pytest results in failed
# assertions, meaning that is is not extremely robust
rng = np.random.RandomState(0)
param_grid = product((1.0, 1e-2), (True, False),
('svd', 'cholesky', 'lsqr', 'sparse_cg'))
for n_samples, n_features in ((6, 5), (5, 10)):
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
sample_weight = 1.0 + rng.rand(n_samples)
for (alpha, intercept, solver) in param_grid:
# Ridge with explicit sample_weight
est = Ridge(alpha=alpha,
fit_intercept=intercept,
solver=solver,
tol=1e-6)
est.fit(X, y, sample_weight=sample_weight)
coefs = est.coef_
inter = est.intercept_
# Closed form of the weighted regularized least square
# theta = (X^T W X + alpha I)^(-1) * X^T W y
W = np.diag(sample_weight)
if intercept is False:
X_aug = X
I = np.eye(n_features)
else:
dummy_column = np.ones(shape=(n_samples, 1))
X_aug = np.concatenate((dummy_column, X), axis=1)
I = np.eye(n_features + 1)
I[0, 0] = 0
cf_coefs = linalg.solve(
X_aug.T.dot(W).dot(X_aug) + alpha * I,
X_aug.T.dot(W).dot(y))
if intercept is False:
assert_array_almost_equal(coefs, cf_coefs)
else:
assert_array_almost_equal(coefs, cf_coefs[1:])
assert_almost_equal(inter, cf_coefs[0])
def test_ridge_fit_intercept_sparse():
X, y = make_regression(n_samples=1000, n_features=2, n_informative=2,
bias=10., random_state=42)
X_csr = sp.csr_matrix(X)
dense = Ridge(alpha=1., tol=1.e-15, solver='sag', fit_intercept=True)
sparse = Ridge(alpha=1., tol=1.e-15, solver='sag', fit_intercept=True)
dense.fit(X, y)
sparse.fit(X_csr, y)
assert_almost_equal(dense.intercept_, sparse.intercept_)
assert_array_almost_equal(dense.coef_, sparse.coef_)
# test the solver switch and the corresponding warning
sparse = Ridge(alpha=1., tol=1.e-15, solver='lsqr', fit_intercept=True)
assert_warns(UserWarning, sparse.fit, X_csr, y)
assert_almost_equal(dense.intercept_, sparse.intercept_)
assert_array_almost_equal(dense.coef_, sparse.coef_)
def test_ridge_fit_intercept_sparse_sag():
X, y = _make_sparse_offset_regression(
n_features=5, n_samples=20, random_state=0, X_offset=5.)
X_csr = sp.csr_matrix(X)
params = dict(alpha=1., solver='sag', fit_intercept=True,
tol=1e-10, max_iter=100000)
dense_ridge = Ridge(**params)
sparse_ridge = Ridge(**params)
dense_ridge.fit(X, y)
with pytest.warns(None) as record:
sparse_ridge.fit(X_csr, y)
assert len(record) == 0
assert np.allclose(dense_ridge.intercept_, sparse_ridge.intercept_,
rtol=1e-4)
assert np.allclose(dense_ridge.coef_, sparse_ridge.coef_, rtol=1e-4)
with pytest.warns(UserWarning, match='"sag" solver requires.*'):
Ridge(solver='sag').fit(X_csr, y)
def test_ridge_vs_lstsq():
"""On alpha=0., Ridge and OLS yield the same solution."""
# we need more samples than features
n_samples, n_features = 5, 4
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
ridge = Ridge(alpha=0., fit_intercept=False)
ols = LinearRegression(fit_intercept=False)
ridge.fit(X, y)
ols.fit(X, y)
assert_almost_equal(ridge.coef_, ols.coef_)
ridge.fit(X, y)
ols.fit(X, y)
assert_almost_equal(ridge.coef_, ols.coef_)
def test_ridge_vs_lstsq():
"""On alpha=0., Ridge and OLS yield the same solution."""
# we need more samples than features
n_samples, n_features = 5, 4
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
ridge = Ridge(alpha=0., fit_intercept=False)
ols = LinearRegression(fit_intercept=False)
ridge.fit(X, y)
ols.fit(X, y)
assert_almost_equal(ridge.coef_, ols.coef_)
ridge.fit(X, y)
ols.fit(X, y)
assert_almost_equal(ridge.coef_, ols.coef_)
def test_ridge_sample_weights():
# TODO: loop over sparse data as well
# Note: parametrizing this test with pytest results in failed
# assertions, meaning that is is not extremely robust
rng = np.random.RandomState(0)
param_grid = product((1.0, 1e-2), (True, False),
('svd', 'cholesky', 'lsqr', 'sparse_cg'))
for n_samples, n_features in ((6, 5), (5, 10)):
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
sample_weight = 1.0 + rng.rand(n_samples)
for (alpha, intercept, solver) in param_grid:
# Ridge with explicit sample_weight
est = Ridge(alpha=alpha, fit_intercept=intercept,
solver=solver, tol=1e-6)
est.fit(X, y, sample_weight=sample_weight)
coefs = est.coef_
inter = est.intercept_
# Closed form of the weighted regularized least square
# theta = (X^T W X + alpha I)^(-1) * X^T W y
W = np.diag(sample_weight)
if intercept is False:
X_aug = X
I = np.eye(n_features)
else:
dummy_column = np.ones(shape=(n_samples, 1))
X_aug = np.concatenate((dummy_column, X), axis=1)
I = np.eye(n_features + 1)
I[0, 0] = 0
cf_coefs = linalg.solve(X_aug.T.dot(W).dot(X_aug) + alpha * I,
X_aug.T.dot(W).dot(y))
if intercept is False:
assert_array_almost_equal(coefs, cf_coefs)
else:
assert_array_almost_equal(coefs, cf_coefs[1:])
assert_almost_equal(inter, cf_coefs[0])
def test_toy_ridge_object():
# Test BayesianRegression ridge classifier
# TODO: test also n_samples > n_features
X = np.array([[1], [2]])
Y = np.array([1, 2])
clf = Ridge(alpha=0.0)
clf.fit(X, Y)
X_test = [[1], [2], [3], [4]]
assert_almost_equal(clf.predict(X_test), [1., 2, 3, 4])
assert_equal(len(clf.coef_.shape), 1)
assert_equal(type(clf.intercept_), np.float64)
Y = np.vstack((Y, Y)).T
clf.fit(X, Y)
X_test = [[1], [2], [3], [4]]
assert_equal(len(clf.coef_.shape), 2)
assert_equal(type(clf.intercept_), np.ndarray)
def test_ridge_fit_intercept_sparse():
X, y = make_regression(n_samples=1000, n_features=2, n_informative=2,
bias=10., random_state=42)
X_csr = sp.csr_matrix(X)
for solver in ['sag', 'sparse_cg']:
dense = Ridge(alpha=1., tol=1.e-15, solver=solver, fit_intercept=True)
sparse = Ridge(alpha=1., tol=1.e-15, solver=solver, fit_intercept=True)
dense.fit(X, y)
with pytest.warns(None) as record:
sparse.fit(X_csr, y)
assert len(record) == 0
assert_almost_equal(dense.intercept_, sparse.intercept_)
assert_array_almost_equal(dense.coef_, sparse.coef_)
# test the solver switch and the corresponding warning
for solver in ['saga', 'lsqr']:
sparse = Ridge(alpha=1., tol=1.e-15, solver=solver, fit_intercept=True)
assert_raises_regex(ValueError, "In Ridge,", sparse.fit, X_csr, y)
def test_toy_ridge_object():
# Test BayesianRegression ridge classifier
# TODO: test also n_samples > n_features
X = np.array([[1], [2]])
Y = np.array([1, 2])
reg = Ridge(alpha=0.0)
reg.fit(X, Y)
X_test = [[1], [2], [3], [4]]
assert_almost_equal(reg.predict(X_test), [1., 2, 3, 4])
assert len(reg.coef_.shape) == 1
assert type(reg.intercept_) == np.float64
Y = np.vstack((Y, Y)).T
reg.fit(X, Y)
X_test = [[1], [2], [3], [4]]
assert len(reg.coef_.shape) == 2
assert type(reg.intercept_) == np.ndarray
def test_toy_ridge_object():
# Test BayesianRegression ridge classifier
# TODO: test also n_samples > n_features
X = np.array([[1], [2]])
Y = np.array([1, 2])
clf = Ridge(alpha=0.0)
clf.fit(X, Y)
X_test = [[1], [2], [3], [4]]
assert_almost_equal(clf.predict(X_test), [1., 2, 3, 4])
assert_equal(len(clf.coef_.shape), 1)
assert_equal(type(clf.intercept_), np.float64)
Y = np.vstack((Y, Y)).T
clf.fit(X, Y)
X_test = [[1], [2], [3], [4]]
assert_equal(len(clf.coef_.shape), 2)
assert_equal(type(clf.intercept_), np.ndarray)
def test_ridge_fit_intercept_sparse():
X, y = make_regression(n_samples=1000, n_features=2, n_informative=2,
bias=10., random_state=42)
X_csr = sp.csr_matrix(X)
for solver in ['sag', 'sparse_cg']:
dense = Ridge(alpha=1., tol=1.e-15, solver=solver, fit_intercept=True)
sparse = Ridge(alpha=1., tol=1.e-15, solver=solver, fit_intercept=True)
dense.fit(X, y)
with pytest.warns(None) as record:
sparse.fit(X_csr, y)
assert len(record) == 0
assert_almost_equal(dense.intercept_, sparse.intercept_)
assert_array_almost_equal(dense.coef_, sparse.coef_)
# test the solver switch and the corresponding warning
for solver in ['saga', 'lsqr']:
sparse = Ridge(alpha=1., tol=1.e-15, solver=solver, fit_intercept=True)
assert_raises_regex(ValueError, "In Ridge,", sparse.fit, X_csr, y)
def test_dtype_match(solver):
rng = np.random.RandomState(0)
alpha = 1.0
n_samples, n_features = 6, 5
X_64 = rng.randn(n_samples, n_features)
y_64 = rng.randn(n_samples)
X_32 = X_64.astype(np.float32)
y_32 = y_64.astype(np.float32)
# Check type consistency 32bits
ridge_32 = Ridge(
alpha=alpha,
solver=solver,
max_iter=500,
tol=1e-10,
)
ridge_32.fit(X_32, y_32)
coef_32 = ridge_32.coef_
# Check type consistency 64 bits
ridge_64 = Ridge(
alpha=alpha,
solver=solver,
max_iter=500,
tol=1e-10,
)
ridge_64.fit(X_64, y_64)
coef_64 = ridge_64.coef_
# Do the actual checks at once for easier debug
assert coef_32.dtype == X_32.dtype
assert coef_64.dtype == X_64.dtype
assert ridge_32.predict(X_32).dtype == X_32.dtype
assert ridge_64.predict(X_64).dtype == X_64.dtype
if solver == "sparse_cg" and sys.platform == "darwin":
pytest.xfail(
"Known failures on MacOS, See "
"https://github.com/scikit-learn/scikit-learn/issues/13868")
assert_allclose(ridge_32.coef_, ridge_64.coef_, rtol=1e-4)
def test_raises_value_error_if_sample_weights_greater_than_1d():
"""Sample weights must be either scalar or 1D"""
n_sampless = [2, 3]
n_featuress = [3, 2]
rng = np.random.RandomState(42)
for n_samples, n_features in zip(n_sampless, n_featuress):
X = rng.randn(n_samples, n_features)
y = rng.randn(n_samples)
sample_weights_OK = rng.randn(n_samples) ** 2 + 1
sample_weights_OK_1 = 1.
sample_weights_OK_2 = 2.
sample_weights_not_OK = sample_weights_OK[:, np.newaxis]
sample_weights_not_OK_2 = sample_weights_OK[np.newaxis, :]
ridge = Ridge(alpha=1)
# make sure the "OK" sample weights actually work
ridge.fit(X, y, sample_weights_OK)
ridge.fit(X, y, sample_weights_OK_1)
ridge.fit(X, y, sample_weights_OK_2)
def fit_ridge_not_ok():
ridge.fit(X, y, sample_weights_not_OK)
def fit_ridge_not_ok_2():
ridge.fit(X, y, sample_weights_not_OK_2)
assert_raise_message(ValueError,
"Sample weights must be 1D array or scalar",
fit_ridge_not_ok)
assert_raise_message(ValueError,
"Sample weights must be 1D array or scalar",
fit_ridge_not_ok_2)
class RidgeImpl():
def __init__(self, alpha=1.0, fit_intercept=True, normalize=False, copy_X=True, max_iter=None, tol=0.001, solver='auto', random_state=None):
self._hyperparams = {
'alpha': alpha,
'fit_intercept': fit_intercept,
'normalize': normalize,
'copy_X': copy_X,
'max_iter': max_iter,
'tol': tol,
'solver': solver,
'random_state': random_state}
def fit(self, X, y=None):
self._sklearn_model = SKLModel(**self._hyperparams)
if (y is not None):
self._sklearn_model.fit(X, y)
else:
self._sklearn_model.fit(X)
return self
def predict(self, X):
return self._sklearn_model.predict(X)
def eval_aggr_shifts(X, y, ignore_rows):
eps = 1e-6
pred = []
real = []
for inst_n in ignore_rows:
X = np.concatenate((X[:inst_n], X[inst_n+1:]))
y = np.concatenate((y[:inst_n], y[inst_n+1:]))
n = X.shape[0]
for inst_n in range(n):
x_i = X[inst_n]
y_i = y[inst_n]
X_train = np.concatenate((X[:inst_n], X[inst_n+1:]))
y_train = np.concatenate((y[:inst_n], y[inst_n+1:]))
y_train = np.array([max(eps, min(1 - eps, val)) for val in y_train])
y_train = np.log(y_train / (1 - y_train))
model = Ridge(alpha=.2, fit_intercept=True, normalize=True)
#model = Lasso(alpha=.001, fit_intercept=True, normalize=True)
model.fit(X_train, y_train)
y_hat = model.predict(x_i.reshape(1, -1))[0]
y_i1 = max(eps, min(1 - eps, y_i))
y_i1 = np.log(y_i1 / (1 - y_i1))
print('inst: ' + str(inst_n) + ', prediction: ' + str(y_hat) + ', err: ' + str(y_hat - y_i1))
pred.append(1 / (1 + exp(-y_hat)))
real.append(y_i)
model = Ridge(alpha=.2, fit_intercept=True, normalize=True)
model.fit(X, y)
return pred, real, model.coef_
def test_ridge_fit_intercept_sparse(solver):
X, y = _make_sparse_offset_regression(n_features=20, random_state=0)
X_csr = sp.csr_matrix(X)
# for now only sparse_cg can correctly fit an intercept with sparse X with
# default tol and max_iter.
# sag is tested separately in test_ridge_fit_intercept_sparse_sag
# because it requires more iterations and should raise a warning if default
# max_iter is used.
# other solvers raise an exception, as checked in
# test_ridge_fit_intercept_sparse_error
#
# "auto" should switch to "sparse_cg" when X is sparse
# so the reference we use for both ("auto" and "sparse_cg") is
# Ridge(solver="sparse_cg"), fitted using the dense representation (note
# that "sparse_cg" can fit sparse or dense data)
dense_ridge = Ridge(solver='sparse_cg')
sparse_ridge = Ridge(solver=solver)
dense_ridge.fit(X, y)
with pytest.warns(None) as record:
sparse_ridge.fit(X_csr, y)
assert len(record) == 0
assert np.allclose(dense_ridge.intercept_, sparse_ridge.intercept_)
assert np.allclose(dense_ridge.coef_, sparse_ridge.coef_)
def test_dtype_match(solver):
rng = np.random.RandomState(0)
alpha = 1.0
n_samples, n_features = 6, 5
X_64 = rng.randn(n_samples, n_features)
y_64 = rng.randn(n_samples)
X_32 = X_64.astype(np.float32)
y_32 = y_64.astype(np.float32)
# Check type consistency 32bits
ridge_32 = Ridge(
alpha=alpha,
solver=solver,
max_iter=500,
tol=1e-10,
)
ridge_32.fit(X_32, y_32)
coef_32 = ridge_32.coef_
# Check type consistency 64 bits
ridge_64 = Ridge(
alpha=alpha,
solver=solver,
max_iter=500,
tol=1e-10,
)
ridge_64.fit(X_64, y_64)
coef_64 = ridge_64.coef_
# Do the actual checks at once for easier debug
assert coef_32.dtype == X_32.dtype
assert coef_64.dtype == X_64.dtype
assert ridge_32.predict(X_32).dtype == X_32.dtype
assert ridge_64.predict(X_64).dtype == X_64.dtype
assert_allclose(ridge_32.coef_, ridge_64.coef_, rtol=1e-4)
def test_ridge():
"""Ridge regression convergence test using score
TODO: for this test to be robust, we should use a dataset instead
of np.random.
"""
rng = np.random.RandomState(0)
alpha = 1.0
for solver in ("svd", "sparse_cg", "cholesky", "lsqr"):
# With more samples than features
n_samples, n_features = 6, 5
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
ridge = Ridge(alpha=alpha, solver=solver)
ridge.fit(X, y)
assert_equal(ridge.coef_.shape, (X.shape[1], ))
assert_greater(ridge.score(X, y), 0.47)
if solver == "cholesky":
# Currently the only solver to support sample_weight.
ridge.fit(X, y, sample_weight=np.ones(n_samples))
assert_greater(ridge.score(X, y), 0.47)
# With more features than samples
n_samples, n_features = 5, 10
y = rng.randn(n_samples)
X = rng.randn(n_samples, n_features)
ridge = Ridge(alpha=alpha, solver=solver)
ridge.fit(X, y)
assert_greater(ridge.score(X, y), .9)
if solver == "cholesky":
# Currently the only solver to support sample_weight.
ridge.fit(X, y, sample_weight=np.ones(n_samples))
assert_greater(ridge.score(X, y), 0.9)
def test_ridge_shapes():
# Test shape of coef_ and intercept_
rng = np.random.RandomState(0)
n_samples, n_features = 5, 10
X = rng.randn(n_samples, n_features)
y = rng.randn(n_samples)
Y1 = y[:, np.newaxis]
Y = np.c_[y, 1 + y]
ridge = Ridge()
ridge.fit(X, y)
assert_equal(ridge.coef_.shape, (n_features, ))
assert_equal(ridge.intercept_.shape, ())
ridge.fit(X, Y1)
assert_equal(ridge.coef_.shape, (1, n_features))
assert_equal(ridge.intercept_.shape, (1, ))
ridge.fit(X, Y)
assert_equal(ridge.coef_.shape, (2, n_features))
assert_equal(ridge.intercept_.shape, (2, ))
def test_ridge_shapes():
# Test shape of coef_ and intercept_
rng = np.random.RandomState(0)
n_samples, n_features = 5, 10
X = rng.randn(n_samples, n_features)
y = rng.randn(n_samples)
Y1 = y[:, np.newaxis]
Y = np.c_[y, 1 + y]
ridge = Ridge()
ridge.fit(X, y)
assert_equal(ridge.coef_.shape, (n_features,))
assert_equal(ridge.intercept_.shape, ())
ridge.fit(X, Y1)
assert_equal(ridge.coef_.shape, (1, n_features))
assert_equal(ridge.intercept_.shape, (1, ))
ridge.fit(X, Y)
assert_equal(ridge.coef_.shape, (2, n_features))
assert_equal(ridge.intercept_.shape, (2, ))
def test_sparse_cg_max_iter():
reg = Ridge(solver="sparse_cg", max_iter=1)
reg.fit(X_diabetes, y_diabetes)
assert_equal(reg.coef_.shape[0], X_diabetes.shape[1])
def main(num_pts, num_children, learning_rate=1.5, learning_scale=0.8, rand_seed=0):
top_node = Node(SqLoss, parent=None, name="root", input_dim=0)
child_nodes = [Node(SqLoss, parent=top_node, input_dim=FEATURE_DIM, name='Child {:d}'.format(i))
for i in xrange(num_children)]
#child_nodes = []
# for i in xrange(num_children):
# func = linear_features
# if i % 2 == 0:
# func = square_features
# child_nodes.append(Node(None, parent=top_node, input_dim=FEATURE_DIM, predict_func=func,
# name='Child {:d}'.format(i)))
validation_set = [pt for pt in dataset(500, seed=rand_seed + 1)]
batch_set = [pt for pt in dataset(num_pts, seed=rand_seed)]
from sklearn.linear_model.ridge import Ridge
batch_learner = Ridge(alpha=1e-15, fit_intercept=False)
batch_learner.fit(np.vstack([pt.x for pt in batch_set]), np.array([pt.y for pt in batch_set]))
batch_pred = batch_learner.predict(np.vstack([pt.x for pt in validation_set]))
Yval = np.array([pt.y for pt in validation_set])
# THIS HAS TO BE THE SAME LOSS AS THE TOP NODE!
mean_batch_err = np.mean([top_node.loss(pred, val) for (pred, val) in zip(batch_pred, Yval)])
#err = batch_pred - Yval; mean_batch_err = np.mean(0.5*err*err)
print('Batch err: {:.4g}'.format(mean_batch_err))
npprint = partial(np.array_str, precision=3)
multiprocess = num_children >= 75
if multiprocess:
from pathos.multiprocessing import ProcessingPool as Pool
from pathos.multiprocessing import cpu_count
#p = Pool(int(ceil(0.75*cpu_count())))
p = Pool(cpu_count())
val_helper = partial(predict_layer, child_nodes=child_nodes, top_node=top_node)
learner_weights = np.array([node.w for node in child_nodes])
disp_num_child = 15
if num_children < disp_num_child:
print('Child learner weights: {}'.format(npprint(learner_weights.ravel())))
validation_preds = []
per_iter_learner_weights = []
print 'Starting Online Boosting...'
for i, pt in enumerate(dataset(num_pts, seed=rand_seed)):
per_iter_learner_weights.append(learner_weights)
# Compute loss on Validation set
if multiprocess:
val_results = p.map(val_helper, validation_set)
else:
val_results = [predict_layer(val_pt, child_nodes, top_node) for val_pt in validation_set]
val_psums, val_losses = zip(*val_results)
val_preds = [psum[-1] for psum in val_psums]
validation_preds.append(val_preds)
avg_val_loss = np.mean(val_losses)
# Compute the partial sums, loss on current data point
partial_sums, top_loss = predict_layer(pt, child_nodes, top_node)
# get the gradient of the top loss at each partial sum
true_val = pt.y
offset_partials = partial_sums.copy()
offset_partials[1:] = partial_sums[:-1]
offset_partials[0] = 0
dlosses = [node.dloss(pred_val, true_val) for pred_val, node in zip(offset_partials, child_nodes)]
step_size = learning_scale / np.power((i + 1), learning_rate)
learner_weights = np.array([node.grad_step(pt.x, loss, step_size)
for (node, loss) in zip(child_nodes, dlosses)])
if i < 1 or i == num_pts - 1 or (i < num_children and num_children < disp_num_child)\
or i % min(int(ceil(num_pts * 0.05)), 25) == 0 or avg_val_loss > 1e3:
print('Iteration {:d}/{:d}: (x={:.2g},y={:.2g})'.format(i + 1, num_pts, pt.x, pt.y))
print(' Avg validation loss on pt: {:.4g} vs Batch: {:.4g}'.format(avg_val_loss,
mean_batch_err))
print(' Top layer loss on pt: {:.4g}'.format(top_loss))
if num_children < disp_num_child:
print(' Child learner weights: {}'.format(npprint(learner_weights.ravel())))
print(' Partial sums: {}'.format(npprint(partial_sums)))
print(' Took descent step of step size {:.4g}...'.format(step_size))
# endfor
return validation_set, validation_preds, batch_pred, batch_set, per_iter_learner_weights
def _test_ridge_diabetes(filter_):
ridge = Ridge(fit_intercept=False)
ridge.fit(filter_(X_diabetes), y_diabetes)
return np.round(ridge.score(filter_(X_diabetes), y_diabetes), 5)
def _test_ridge_loo(filter_):
# test that can work with both dense or sparse matrices
n_samples = X_diabetes.shape[0]
ret = []
ridge_gcv = _RidgeGCV(fit_intercept=False)
ridge = Ridge(alpha=1.0, fit_intercept=False)
# generalized cross-validation (efficient leave-one-out)
decomp = ridge_gcv._pre_compute(X_diabetes, y_diabetes)
errors, c = ridge_gcv._errors(1.0, y_diabetes, *decomp)
values, c = ridge_gcv._values(1.0, y_diabetes, *decomp)
# brute-force leave-one-out: remove one example at a time
errors2 = []
values2 = []
for i in range(n_samples):
sel = np.arange(n_samples) != i
X_new = X_diabetes[sel]
y_new = y_diabetes[sel]
ridge.fit(X_new, y_new)
value = ridge.predict([X_diabetes[i]])[0]
error = (y_diabetes[i] - value) ** 2
errors2.append(error)
values2.append(value)
# check that efficient and brute-force LOO give same results
assert_almost_equal(errors, errors2)
assert_almost_equal(values, values2)
# generalized cross-validation (efficient leave-one-out,
# SVD variation)
decomp = ridge_gcv._pre_compute_svd(X_diabetes, y_diabetes)
errors3, c = ridge_gcv._errors_svd(ridge.alpha, y_diabetes, *decomp)
values3, c = ridge_gcv._values_svd(ridge.alpha, y_diabetes, *decomp)
# check that efficient and SVD efficient LOO give same results
assert_almost_equal(errors, errors3)
assert_almost_equal(values, values3)
# check best alpha
ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
alpha_ = ridge_gcv.alpha_
ret.append(alpha_)
# check that we get same best alpha with custom loss_func
f = ignore_warnings
scoring = make_scorer(mean_squared_error, greater_is_better=False)
ridge_gcv2 = RidgeCV(fit_intercept=False, scoring=scoring)
f(ridge_gcv2.fit)(filter_(X_diabetes), y_diabetes)
assert_equal(ridge_gcv2.alpha_, alpha_)
# check that we get same best alpha with custom score_func
func = lambda x, y: -mean_squared_error(x, y)
scoring = make_scorer(func)
ridge_gcv3 = RidgeCV(fit_intercept=False, scoring=scoring)
f(ridge_gcv3.fit)(filter_(X_diabetes), y_diabetes)
assert_equal(ridge_gcv3.alpha_, alpha_)
# check that we get same best alpha with a scorer
scorer = get_scorer('mean_squared_error')
ridge_gcv4 = RidgeCV(fit_intercept=False, scoring=scorer)
ridge_gcv4.fit(filter_(X_diabetes), y_diabetes)
assert_equal(ridge_gcv4.alpha_, alpha_)
# check that we get same best alpha with sample weights
ridge_gcv.fit(filter_(X_diabetes), y_diabetes,
sample_weight=np.ones(n_samples))
assert_equal(ridge_gcv.alpha_, alpha_)
# simulate several responses
Y = np.vstack((y_diabetes, y_diabetes)).T
ridge_gcv.fit(filter_(X_diabetes), Y)
Y_pred = ridge_gcv.predict(filter_(X_diabetes))
ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
y_pred = ridge_gcv.predict(filter_(X_diabetes))
assert_array_almost_equal(np.vstack((y_pred, y_pred)).T,
Y_pred, decimal=5)
return ret
train0 = extract_predictor(train_dataset, True)
train1 = extract_target(train_dataset)
test0 = extract_predictor(test_dataset, False)
results = []
for cnt in range(1000):
projected0 = []
projected1 = []
for i in xrange(len(train0)):
if random.random() < 0.4:
continue
projected0.append(train0[i])
projected1.append(train1[i])
print "now fitting the model", cnt, "with len", len(projected0)
model = Ridge()
model.fit(projected0, projected1)
predictions=model.predict(test0)
results.append(list(predictions))
final_result = []
for ind in xrange(len(results[0])):
cand = []
for i in xrange(len(results)):
cand.append(results[i][ind])
final_result.append(sum(sorted(cand)[100:-100])*1.0/(len(cand)-200))
#predictions=model.predict(valid_dataset)
#Evaluate the quality of the prediction
#print sklearn.metrics.mean_absolute_error(predictions,valid_target)
model2_train1 += virtual_test1
print "now saving the result"
ff = open('virtual_train_data.json', 'w')
ff.write(json.dumps([model2_train0, model2_train1]))
ff.close()
if sys.argv[1] == "second":
ff = open('virtual_train_data.json', 'r')
model2_train0, model2_train1 = json.loads(ff.read())
ff.close()
print "opened train0 and train1 with each length", len(model2_train0), len(model2_train1)
print model2_train0[0]
print model2_train1[0]
ff = open('intermediate_result.json', 'r')
model2_test0, _ = json.loads(ff.read())
print model2_test0[0]
model2 = Ridge()
print "start fitting 2nd model"
model2.fit(model2_train0, model2_train1)
print "start predicting"
predictions=model2.predict(model2_test0)
print "saving the predicted result into the file"
f = open('result.csv', 'w')
f.write("ID;COTIS\n");
for ind, prd in enumerate(predictions):
f.write(my_ids[ind] + ';' + str(prd) + '\n')
f.close()
print "all tasks completed"
#print(krrs_trigpredictions)
for lst in krrs_trigpredictions:
print(compute_MSE(np.array(lst).reshape(200, 1), test_y.reshape(200, 1)))
berr_polypredictions = []
for deg in [1, 2, 4, 6]:
train_x_new = []
for i in range(train_x.shape[0]):
train_x_new.append([train_x[i]**j for j in range(deg + 1)])
test_x_new = []
for i in range(test_x.shape[0]):
test_x_new.append([test_x[i]**j for j in range(deg + 1)])
clf = Ridge()
clf.fit(train_x_new, train_y)
pred_y = clf.predict(test_x_new)
#print(pred_y)
berr_polypredictions.append(pred_y)
#print(berr_polypredictions)
for lst in berr_polypredictions:
print(compute_MSE(np.array(lst).reshape(200, 1), test_y.reshape(200, 1)))
def trig_expansion(train_x, degree):
res = [1]
d = 0.5
for de in range(degree):
res.append(np.sin(de * d * train_x))
res.append(np.cos(de * d * train_x))
def test_sparse_cg_max_iter():
reg = Ridge(solver="sparse_cg", max_iter=1)
reg.fit(X_diabetes, y_diabetes)
assert_equal(reg.coef_.shape[0], X_diabetes.shape[1])
def _test_ridge_diabetes(filter_):
ridge = Ridge(fit_intercept=False)
ridge.fit(filter_(X_diabetes), y_diabetes)
return np.round(ridge.score(filter_(X_diabetes), y_diabetes), 5)
def _test_ridge_loo(filter_):
# test that can work with both dense or sparse matrices
n_samples = X_diabetes.shape[0]
ret = []
ridge_gcv = _RidgeGCV(fit_intercept=False)
ridge = Ridge(alpha=1.0, fit_intercept=False)
# generalized cross-validation (efficient leave-one-out)
decomp = ridge_gcv._pre_compute(X_diabetes, y_diabetes)
errors, c = ridge_gcv._errors(1.0, y_diabetes, *decomp)
values, c = ridge_gcv._values(1.0, y_diabetes, *decomp)
# brute-force leave-one-out: remove one example at a time
errors2 = []
values2 = []
for i in range(n_samples):
sel = np.arange(n_samples) != i
X_new = X_diabetes[sel]
y_new = y_diabetes[sel]
ridge.fit(X_new, y_new)
value = ridge.predict([X_diabetes[i]])[0]
error = (y_diabetes[i] - value)**2
errors2.append(error)
values2.append(value)
# check that efficient and brute-force LOO give same results
assert_almost_equal(errors, errors2)
assert_almost_equal(values, values2)
# generalized cross-validation (efficient leave-one-out,
# SVD variation)
decomp = ridge_gcv._pre_compute_svd(X_diabetes, y_diabetes)
errors3, c = ridge_gcv._errors_svd(ridge.alpha, y_diabetes, *decomp)
values3, c = ridge_gcv._values_svd(ridge.alpha, y_diabetes, *decomp)
# check that efficient and SVD efficient LOO give same results
assert_almost_equal(errors, errors3)
assert_almost_equal(values, values3)
# check best alpha
ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
alpha_ = ridge_gcv.alpha_
ret.append(alpha_)
# check that we get same best alpha with custom loss_func
f = ignore_warnings
scoring = make_scorer(mean_squared_error, greater_is_better=False)
ridge_gcv2 = RidgeCV(fit_intercept=False, scoring=scoring)
f(ridge_gcv2.fit)(filter_(X_diabetes), y_diabetes)
assert_equal(ridge_gcv2.alpha_, alpha_)
# check that we get same best alpha with custom score_func
func = lambda x, y: -mean_squared_error(x, y)
scoring = make_scorer(func)
ridge_gcv3 = RidgeCV(fit_intercept=False, scoring=scoring)
f(ridge_gcv3.fit)(filter_(X_diabetes), y_diabetes)
assert_equal(ridge_gcv3.alpha_, alpha_)
# check that we get same best alpha with a scorer
scorer = get_scorer('mean_squared_error')
ridge_gcv4 = RidgeCV(fit_intercept=False, scoring=scorer)
ridge_gcv4.fit(filter_(X_diabetes), y_diabetes)
assert_equal(ridge_gcv4.alpha_, alpha_)
# check that we get same best alpha with sample weights
ridge_gcv.fit(filter_(X_diabetes),
y_diabetes,
sample_weight=np.ones(n_samples))
assert_equal(ridge_gcv.alpha_, alpha_)
# simulate several responses
Y = np.vstack((y_diabetes, y_diabetes)).T
ridge_gcv.fit(filter_(X_diabetes), Y)
Y_pred = ridge_gcv.predict(filter_(X_diabetes))
ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
y_pred = ridge_gcv.predict(filter_(X_diabetes))
assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=5)
return ret
def trainModel(param,feat_folder,feat_name):
#read data from folder
print 'now we read data from folder:%s'%(feat_folder)
#start cv
print 'now we need to generate cross_validation'
accuracy_cv = []
for i in range(0,2):
print 'this is the run:%d cross-validation'%(i+1)
testIndex = loadCVIndex("%s/test.run%d.txt"%("../data/feat/combine",(i+1)))
#if we use xgboost to train model ,we need to use svmlib format
if param['task'] in ['regression']:
#with xgb we will dump the file with CV,and we will read data
train_data = xgb.DMatrix("%s/run%d/train.svm.txt"%(feat_folder,(i+1)))
valid_data = xgb.DMatrix("%s/run%d/test.svm.txt"%(feat_folder,(i+1)))
watchlist = [(train_data,'train'),(valid_data,'valid')]
bst = xgb.train(param,train_data,int(param['num_round']),watchlist)
pred = bst.predict(valid_data)
elif param['task'] in ['clf_skl_lr']:
train_data,train_label = load_svmlight_file("%s/run%d/train.svm.txt"%(feat_folder,(i+1)))
test_data,test_label = load_svmlight_file("%s/run%d/test.svm.txt"%(feat_folder,(i+1)))
train_data = train_data.tocsr()
test_data = test_data.tocsr()
clf = LogisticRegression()
clf.fit(train_data,train_label)
pred = clf.predict(test_data)
elif param['task'] == "reg_skl_rf":
## regression with sklearn random forest regressor
train_data,train_label = load_svmlight_file("%s/run%d/train.svm.txt"%(feat_folder,(i+1)))
test_data,test_label = load_svmlight_file("%s/run%d/test.svm.txt"%(feat_folder,(i+1)))
rf = RandomForestRegressor(n_estimators=param['n_estimators'],
max_features=param['max_features'],
n_jobs=param['n_jobs'],
random_state=param['random_state'])
rf.fit(train_data, test_label)
pred = rf.predict(test_data)
elif param['task'] == "reg_skl_etr":
## regression with sklearn extra trees regressor
train_data,train_label = load_svmlight_file("%s/run%d/train.svm.txt"%(feat_folder,(i+1)))
test_data,test_label = load_svmlight_file("%s/run%d/test.svm.txt"%(feat_folder,(i+1)))
etr = ExtraTreesRegressor(n_estimators=param['n_estimators'],
max_features=param['max_features'],
n_jobs=param['n_jobs'],
random_state=param['random_state'])
etr.fit(train_data,test_label)
pred = etr.predict(test_data)
elif param['task'] in ['reg_skl_gbm'] :
train_data,train_label = load_svmlight_file("%s/run%d/train.svm.txt"%(feat_folder,(i+1)))
test_data,test_label = load_svmlight_file("%s/run%d/test.svm.txt"%(feat_folder,(i+1)))
gbm = GradientBoostingClassifier(n_estimators=int(param['n_estimators']),
learning_rate=param['learning_rate'],
max_features=param['max_features'],
max_depth=param['max_depth'],
subsample=param['subsample'],
random_state=param['random_state'])
feat_names.remove('cid')
gbm.fit(train_data,train_label)
pred = gbm.predict(test_data)
elif param['task'] in ['reg_skl_ridge']:
train_data,train_label = load_svmlight_file("%s/run%d/train.svm.txt"%(feat_folder,(i+1)))
test_data,test_label = load_svmlight_file("%s/run%d/test.svm.txt"%(feat_folder,(i+1)))
train_data = train_data.tocsr()
test_data = test_data.tocsr()
ridge = Ridge(alpha=param["alpha"], normalize=True)
ridge.fit(train_data,train_label)
predraw = ridge.predict(test_data)
print predraw
predrank = predraw.argsort().argsort()
trainIndex = loadCVIndex("%s/train.run%d.txt"%("../data/feat/combine",(i+1)))
cdf = creatCDF(train, trainIndex)
pred = getScore(predrank,cdf)
print pred
"""
elif param['task'] in ['regression']:
elif param['task'] in ['reg_skl_gbm'] :
gbm = GradientBoostingClassifier(n_estimators=int(param['n_estimators']),
learning_rate=param['learning_rate'],
max_features=param['max_features'],
max_depth=param['max_depth'],
subsample=param['subsample'],
random_state=param['random_state'])
feat_names.remove('cid')
gbm.fit(train_data[feat_names],train_data['cid'])
pred = gbm.predict(valid_data[feat_names])
elif param['task'] in ['reg_skl_ridge']:
feat_names.remove('cid')
ridge = Ridge(alpha=param["alpha"], normalize=True)
ridge.fit(train_data[feat_names],train_data['cid'])
pred = ridge.predict(valid_data[feat_names])
"""
#now we use the the accuracy to limit our model
acc = accuracy_model(pred,train.iloc[testIndex]['cid'])
print "the model accurary:%s"%(acc)
accuracy_cv.append(acc)
#here we will count the
accuracy_cv_mean = np.mean(accuracy_cv)
accuracy_cv_std = np.std(accuracy_cv)
print 'the accuracy for %.6f'%(accuracy_cv_mean)
return {'loss':-accuracy_cv_mean,'attachments':{'std':accuracy_cv_std},'status': STATUS_OK}
