def homotopy_path(X, Y, lambda_, coef, y_t, epsilon=1e-3, nu=1.):
eps_0 = epsilon / 10.
step_size = np.sqrt(2. * (epsilon - eps_0) / nu)
Y_t = np.array(list(Y[:-1]) + [y_t], order='F')
y_stop = Y[-1]
while y_t < y_stop:
y_t = min(y_t + step_size, y_stop)
Y_t[-1] = y_t
tol = eps_0 / np.linalg.norm(Y_t) ** 2
alpha = [lambda_ / X.shape[0]]
res = lasso_path(X, Y_t, alphas=alpha, coef_init=coef, eps=tol)
coef = res[1].ravel()
while y_t > y_stop:
y_t = max(y_t - step_size, y_stop)
Y_t[-1] = y_t
tol = eps_0 / np.linalg.norm(Y_t) ** 2
alpha = [lambda_ / X.shape[0]]
res = lasso_path(X, Y_t, alphas=alpha, coef_init=coef, eps=tol)
coef = res[1].ravel()
return coef
def test_lasso_lars_vs_lasso_cd_ill_conditioned():
# Test lasso lars on a very ill-conditioned design, and check that
# it does not blow up, and stays somewhat close to a solution given
# by the coordinate descent solver
# Also test that lasso_path (using lars_path output style) gives
# the same result as lars_path and previous lasso output style
# under these conditions.
rng = np.random.RandomState(42)
# Generate data
n, m = 70, 100
k = 5
X = rng.randn(n, m)
w = np.zeros((m, 1))
i = np.arange(0, m)
rng.shuffle(i)
supp = i[:k]
w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1)
y = np.dot(X, w)
sigma = 0.2
y += sigma * rng.rand(*y.shape)
y = y.squeeze()
f = assert_warns_message
def in_warn_message(msg):
return 'Early stopping' in msg or 'Dropping a regressor' in msg
lars_alphas, _, lars_coef = f(ConvergenceWarning,
in_warn_message,
linear_model.lars_path,
X,
y,
method='lasso')
with ignore_warnings():
_, lasso_coef2, _ = linear_model.lasso_path(X,
y,
alphas=lars_alphas,
tol=1e-6,
fit_intercept=False)
lasso_coef = np.zeros((w.shape[0], len(lars_alphas)))
iter_models = enumerate(
linear_model.lasso_path(X,
y,
alphas=lars_alphas,
tol=1e-6,
return_models=True,
fit_intercept=False))
for i, model in iter_models:
lasso_coef[:, i] = model.coef_
np.testing.assert_array_almost_equal(lars_coef, lasso_coef, decimal=1)
np.testing.assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)
np.testing.assert_array_almost_equal(lasso_coef, lasso_coef2, decimal=1)
def test_lasso_lars_vs_lasso_cd_ill_conditioned():
# Test lasso lars on a very ill-conditioned design, and check that
# it does not blow up, and stays somewhat close to a solution given
# by the coordinate descent solver
# Also test that lasso_path (using lars_path output style) gives
# the same result as lars_path and previous lasso output style
# under these conditions.
rng = np.random.RandomState(42)
# Generate data
n, m = 80, 100
k = 5
X = rng.randn(n, m)
w = np.zeros((m, 1))
i = np.arange(0, m)
rng.shuffle(i)
supp = i[:k]
w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1)
y = np.dot(X, w)
sigma = 0.2
y += sigma * rng.rand(*y.shape)
y = y.squeeze()
with warnings.catch_warnings(record=True) as warning_list:
warnings.simplefilter("always", UserWarning)
lars_alphas, _, lars_coef = linear_model.lars_path(X,
y,
method='lasso')
assert_true(len(warning_list) > 0)
assert_true(('Dropping a regressor' in warning_list[0].message.args[0])
or ('Early stopping' in warning_list[0].message.args[0]))
_, lasso_coef2, _ = linear_model.lasso_path(X,
y,
alphas=lars_alphas,
tol=1e-6,
fit_intercept=False)
lasso_coef = np.zeros((w.shape[0], len(lars_alphas)))
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
for i, model in enumerate(
linear_model.lasso_path(X,
y,
alphas=lars_alphas,
tol=1e-6,
return_models=True,
fit_intercept=False)):
lasso_coef[:, i] = model.coef_
np.testing.assert_array_almost_equal(lars_coef, lasso_coef, decimal=1)
np.testing.assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)
np.testing.assert_array_almost_equal(lasso_coef, lasso_coef2, decimal=1)
def test_lasso_lars_vs_lasso_cd_ill_conditioned():
# Test lasso lars on a very ill-conditioned design, and check that
# it does not blow up, and stays somewhat close to a solution given
# by the coordinate descent solver
# Also test that lasso_path (using lars_path output style) gives
# the same result as lars_path and previous lasso output style
# under these conditions.
rng = np.random.RandomState(42)
# Generate data
n, m = 80, 100
k = 5
X = rng.randn(n, m)
w = np.zeros((m, 1))
i = np.arange(0, m)
rng.shuffle(i)
supp = i[:k]
w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1)
y = np.dot(X, w)
sigma = 0.2
y += sigma * rng.rand(*y.shape)
y = y.squeeze()
with warnings.catch_warnings(record=True) as warning_list:
warnings.simplefilter("always", UserWarning)
lars_alphas, _, lars_coef = linear_model.lars_path(X, y,
method='lasso')
assert_true(len(warning_list) > 0)
assert_true(('Dropping a regressor' in warning_list[0].message.args[0])
or ('Early stopping' in warning_list[0].message.args[0]))
_, lasso_coef2, _ = linear_model.lasso_path(X, y,
alphas=lars_alphas, tol=1e-6,
fit_intercept=False)
lasso_coef = np.zeros((w.shape[0], len(lars_alphas)))
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
for i, model in enumerate(linear_model.lasso_path(X, y,
alphas=lars_alphas,
tol=1e-6,
return_models=True,
fit_intercept=False)):
lasso_coef[:, i] = model.coef_
np.testing.assert_array_almost_equal(lars_coef, lasso_coef, decimal=1)
np.testing.assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)
np.testing.assert_array_almost_equal(lasso_coef, lasso_coef2, decimal=1)
def test_lasso_lars_vs_lasso_cd_ill_conditioned():
# Test lasso lars on a very ill-conditioned design, and check that
# it does not blow up, and stays somewhat close to a solution given
# by the coordinate descent solver
# Also test that lasso_path (using lars_path output style) gives
# the same result as lars_path and previous lasso output style
# under these conditions.
rng = np.random.RandomState(42)
# Generate data
n, m = 70, 100
k = 5
X = rng.randn(n, m)
w = np.zeros((m, 1))
i = np.arange(0, m)
rng.shuffle(i)
supp = i[:k]
w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1)
y = np.dot(X, w)
sigma = 0.2
y += sigma * rng.rand(*y.shape)
y = y.squeeze()
f = assert_warns_message
def in_warn_message(msg):
return 'Early stopping' in msg or 'Dropping a regressor' in msg
lars_alphas, _, lars_coef = f(ConvergenceWarning,
in_warn_message,
linear_model.lars_path, X, y, method='lasso')
with ignore_warnings():
_, lasso_coef2, _ = linear_model.lasso_path(X, y,
alphas=lars_alphas,
tol=1e-6,
fit_intercept=False)
lasso_coef = np.zeros((w.shape[0], len(lars_alphas)))
iter_models = enumerate(linear_model.lasso_path(X, y,
alphas=lars_alphas,
tol=1e-6,
return_models=True,
fit_intercept=False))
for i, model in iter_models:
lasso_coef[:, i] = model.coef_
np.testing.assert_array_almost_equal(lars_coef, lasso_coef, decimal=1)
np.testing.assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)
np.testing.assert_array_almost_equal(lasso_coef, lasso_coef2, decimal=1)
def fit_model(X, Y_t, coef, lambda_, eps_0, method="lasso"):
if coef is None:
coef = np.zeros(X.shape[1])
if method is "lasso":
tol = eps_0 / np.linalg.norm(Y_t)**2
lmd = [lambda_ / X.shape[0]]
res = lasso_path(X,
Y_t,
alphas=lmd,
coef_init=coef,
eps=tol,
max_iter=int(1e8))
coef = res[1].ravel()
elif method is "ridge":
reg = Ridge(alpha=lambda_, fit_intercept=False, solver="auto")
reg.fit(X, Y_t)
coef = reg.coef_
elif method is "logcosh":
# I cannot early stop scipy.minimize with duality gap :-/
coef = logcosh_reg(X, Y_t, lambda_, coef)
elif method is "linex":
# I cannot early stop scipy.minimize with duality gap :-/
coef = linex_reg(X, Y_t, lambda_, coef=coef)
mu = X.dot(coef)
return mu, np.abs(Y_t - mu), coef
def plot(X, y):
#X /= X.std(axis=0) # Standardize data (easier to set the l1_ratio parameter)
# Compute paths
eps = 5e-3 # the smaller it is the longer is the path
alphas_lasso, coefs_lasso, _ = linear_model.lasso_path(X, y, eps)
print alphas_lasso
print coefs_lasso.shape
for i in coefs_lasso.T:
print i
plt.figure(1)
ax = plt.gca()
colors = cycle(['r', 'g', 'b', 'c', 'm', 'y'])
neg_log_alphas_lasso = -np.log10(alphas_lasso)
for coef_l, c in zip(coefs_lasso, colors):
l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c)
plt.xlabel('-Log(alpha)')
plt.ylabel('coefficients')
plt.title('Lasso and Elastic-Net Paths')
#plt.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left')
plt.axis('tight')
plt.show()
def LassoDiff(clf, A, time, p, delta_p, SNR, i=None):
flux = np.sin(2 * np.pi / p * time) + np.sin(2 * np.pi /
(p + delta_p) * time)
flux += np.random.normal(0, flux.std() / np.sqrt(SNR), flux.size)
convWarning = False
with warnings.catch_warnings(record=True) as w:
# clf.fit(A, flux)
clf = linear_model.lasso_path(A, flux)
if len(w) == 1 and type(w[0].message) is ConvergenceWarning:
convWarning = True
# ipdb.set_trace()
coeffs = clf[1][:, 99]
power = coeffs[:A.shape[1] // 2]**2 + coeffs[A.shape[1] // 2:A.shape[1] //
2 * 2]**2
peaks = findPeaks(power)
if peaks.size > 1:
d2P_dp2 = np.gradient(np.gradient(power))
p1_i, p2_i = peaks[-4:][d2P_dp2[peaks[-4:]].argsort()][:2]
else:
p1_i, p2_i = power.argsort()[-2:]
return i, power, p1_i, p2_i, convWarning
def plot_results(self):
"""Create the base regression plots as well as a regularization path plot."""
rc.REG.plot_results(self)
path = linear_model.lasso_path(self.independentVar, self.dependentVar, return_models=False, fit_intercept=False)
alphas = path[0] #Vector of alphas
coefs = (path[1]).T #Array of coefficients for each alpha
viz.plot_regPath(alphas, coefs).plot()
def alpha_choice_fig(x, y, my_alphas, nb_features, train_size):
'''
Parameters
----------
x : data.
y : desired output.
my_alphas : array of different values for alpha.
nb_features : number of features.
train_size : number of train points.
Returns : representation of lasso path
-------
'''
X_train, X_test, y_train, y_test = train_test_split(x, y, train_size=train_size)
alpha_for_path, coefs_lasso, _ = lasso_path(X_train[:, 0:nb_features-1],
X_train[:, nb_features-1],alphas=my_alphas)
for i in range(coefs_lasso.shape[0]):
plt.plot(alpha_for_path, coefs_lasso[i, :])
plt.xlabel('Alpha')
plt.ylabel('Coefficients')
plt.title('Lasso path')
plt.show()
def test_lasso_lars_vs_lasso_cd_ill_conditioned():
# Test lasso lars on a very ill-conditioned design, and check that
# it does not blow up, and stays somewhat close to a solution given
# by the coordinate descent solver
# Also test that lasso_path (using lars_path output style) gives
# the same result as lars_path and previous lasso output style
# under these conditions.
rng = np.random.RandomState(42)
# Generate data
n, m = 70, 100
k = 5
X = rng.randn(n, m)
w = np.zeros((m, 1))
i = np.arange(0, m)
rng.shuffle(i)
supp = i[:k]
w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1)
y = np.dot(X, w)
sigma = 0.2
y += sigma * rng.rand(*y.shape)
y = y.squeeze()
lars_alphas, _, lars_coef = linear_model.lars_path(X, y, method='lasso')
_, lasso_coef2, _ = linear_model.lasso_path(X, y,
alphas=lars_alphas,
tol=1e-6,
fit_intercept=False)
assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)
def meta_algorithm(XTX, XTXi, sampler):
min_success = 6
ntries = 10
def _alpha_grid(X, y, center, XTX):
n, p = X.shape
alphas, coefs, _ = lasso_path(X.copy(),
y.copy(),
Xy=center.copy(),
precompute=XTX.copy())
nselected = np.count_nonzero(coefs, axis=0)
alphas = alphas[nselected < 20]
return alphas
alpha_grid = _alpha_grid(X, y, sampler.center, XTX)
success = np.zeros((p, alpha_grid.shape[0]))
for _ in range(ntries):
scale = 1. # corresponds to sub-samples of 50%
noisy_S = sampler(scale=scale)
_, coefs, _ = lasso_path(X,
y,
Xy=noisy_S,
precompute=XTX,
alphas=alpha_grid)
success += np.abs(np.sign(coefs))
selected = np.apply_along_axis(
lambda row: any(x > min_success for x in row), 1, success)
vars = set(np.nonzero(selected)[0])
return vars
def test_Lasso_Path(self):
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
X /= X.std(axis=0)
df = pdml.ModelFrame(diabetes)
df.data /= df.data.std(axis=0, ddof=False)
self.assert_numpy_array_almost_equal(df.data.values, X)
eps = 5e-3
expected = lm.lasso_path(X, y, eps, fit_intercept=False)
result = df.lm.lasso_path(eps=eps, fit_intercept=False)
self.assert_numpy_array_almost_equal(expected[0], result[0])
self.assert_numpy_array_almost_equal(expected[1], result[1])
self.assert_numpy_array_almost_equal(expected[2], result[2])
expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False)
result = df.lm.enet_path(eps=eps, l1_ratio=0.8, fit_intercept=False)
self.assert_numpy_array_almost_equal(expected[0], result[0])
self.assert_numpy_array_almost_equal(expected[1], result[1])
self.assert_numpy_array_almost_equal(expected[2], result[2])
expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False)
result = df.lm.enet_path(eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False)
self.assert_numpy_array_almost_equal(expected[0], result[0])
self.assert_numpy_array_almost_equal(expected[1], result[1])
self.assert_numpy_array_almost_equal(expected[2], result[2])
expected = lm.lars_path(X, y, method='lasso', verbose=True)
result = df.lm.lars_path(method='lasso', verbose=True)
self.assert_numpy_array_almost_equal(expected[0], result[0])
self.assert_numpy_array_almost_equal(expected[1], result[1])
self.assert_numpy_array_almost_equal(expected[2], result[2])
def test_lasso_lars_vs_lasso_cd_ill_conditioned():
# Test lasso lars on a very ill-conditioned design, and check that
# it does not blow up, and stays somewhat close to a solution given
# by the coordinate descent solver
# Also test that lasso_path (using lars_path output style) gives
# the same result as lars_path and previous lasso output style
# under these conditions.
rng = np.random.RandomState(42)
# Generate data
n, m = 70, 100
k = 5
X = rng.randn(n, m)
w = np.zeros((m, 1))
i = np.arange(0, m)
rng.shuffle(i)
supp = i[:k]
w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1)
y = np.dot(X, w)
sigma = 0.2
y += sigma * rng.rand(*y.shape)
y = y.squeeze()
lars_alphas, _, lars_coef = linear_model.lars_path(X, y, method='lasso')
_, lasso_coef2, _ = linear_model.lasso_path(X,
y,
alphas=lars_alphas,
tol=1e-6,
fit_intercept=False)
assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)
def sklearn_lasso_coefs_plot(X, y):
""" Show the path taken by coefficients as Lasso shrinks them towards zero.
Adapted from: https://scikit-learn.org/stable/auto_examples/linear_model/plot_lasso_coordinate_descent_path.html """
from itertools import cycle
from sklearn.linear_model import lasso_path
eps = 5e-3 # the smaller it is the longer is the path
print("Computing regularization path using the lasso...")
alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept=True)
# Display results
fig, ax = plt.subplots(figsize=(10, 5), dpi=200)
colors = cycle(['b', 'r', 'g', 'c', 'k'])
neg_log_alphas_lasso = -np.log10(alphas_lasso)
for coef, label, c in zip(coefs_lasso, X.columns, colors):
_ = plt.plot(neg_log_alphas_lasso, coef, label=label, c=c)
plt.xlabel('-Log(alpha)')
plt.ylabel('coefficients')
plt.title('Lasso path')
plt.legend(loc='upper left')
plt.axis('tight')
plt.show()
def test_lasso_lars_vs_lasso_cd_ill_conditioned():
# Test lasso lars on a very ill-conditioned design, and check that
# it does not blow up, and stays somewhat close to a solution given
# by the coordinate descent solver
rng = np.random.RandomState(42)
# Generate data
n, m = 80, 100
k = 5
X = rng.randn(n, m)
w = np.zeros((m, 1))
i = np.arange(0, m)
rng.shuffle(i)
supp = i[:k]
w[supp] = np.sign(rng.randn(k, 1)) * (rng.rand(k, 1) + 1)
y = np.dot(X, w)
sigma = 0.2
y += sigma * rng.rand(*y.shape)
y = y.squeeze()
with warnings.catch_warnings(record=True) as warning_list:
warnings.simplefilter("always", UserWarning)
lars_alphas, _, lars_coef = linear_model.lars_path(X, y, method="lasso")
assert_true(len(warning_list) > 0)
assert_true(
("Dropping a regressor" in warning_list[0].message.args[0])
or ("Early stopping" in warning_list[0].message.args[0])
)
lasso_coef = np.zeros((w.shape[0], len(lars_alphas)))
for i, model in enumerate(linear_model.lasso_path(X, y, alphas=lars_alphas, tol=1e-6)):
lasso_coef[:, i] = model.coef_
np.testing.assert_array_almost_equal(lars_coef, lasso_coef, decimal=1)
def test_Lasso_Path(self):
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
X /= X.std(axis=0)
df = pdml.ModelFrame(diabetes)
df.data /= df.data.std(axis=0, ddof=False)
self.assert_numpy_array_almost_equal(df.data.values, X)
eps = 5e-3
expected = lm.lasso_path(X, y, eps, fit_intercept=False)
result = df.lm.lasso_path(eps=eps, fit_intercept=False)
self.assert_numpy_array_almost_equal(expected[0], result[0])
self.assert_numpy_array_almost_equal(expected[1], result[1])
self.assert_numpy_array_almost_equal(expected[2], result[2])
expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False)
result = df.lm.enet_path(eps=eps, l1_ratio=0.8, fit_intercept=False)
self.assert_numpy_array_almost_equal(expected[0], result[0])
self.assert_numpy_array_almost_equal(expected[1], result[1])
self.assert_numpy_array_almost_equal(expected[2], result[2])
expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False)
result = df.lm.enet_path(eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False)
self.assert_numpy_array_almost_equal(expected[0], result[0])
self.assert_numpy_array_almost_equal(expected[1], result[1])
self.assert_numpy_array_almost_equal(expected[2], result[2])
expected = lm.lars_path(X, y, method='lasso', verbose=True)
result = df.lm.lars_path(method='lasso', verbose=True)
self.assert_numpy_array_almost_equal(expected[0], result[0])
self.assert_numpy_array_almost_equal(expected[1], result[1])
self.assert_numpy_array_almost_equal(expected[2], result[2])
def _fit_bootstrap_sample(X, y, func, L):
""" Computes the regularization path for the regression y ~ X.
Parameters
----------
X : array, shape (n_samples , n_features)
y : array, shape (n_samples)
func : string
the function used for computing the regularization path
(either 'lasso', 'elasticnet', or 'lars').
L : int
length of the path.
Returns
-------
array, shape (n_features , L)
0 if the coefficient is null and 1 otherwise.
"""
if func == 'lasso':
_, coef_path, _ = lasso_path(X, y, n_alphas=L)
elif func == 'elasticnet':
_, coef_path, _ = enet_path(X, y, nalphas=L)
elif func == 'lars':
_, _, coef_path = lars_path(X, y, max_iter=L - 1)
return 1 * (coef_path != 0)
def conf_pred(X, Y_seen, lambda_, Y_range, alpha=0.1, method="lasso"):
X_train, X_test, Y_train, Y_test = train_test_split(X[:-1, :],
Y_seen,
test_size=0.5,
random_state=414)
# Training
if method is "lasso":
lmd = [lambda_ / X_train.shape[0]]
res = lasso_path(X_train, Y_train, alphas=lmd, eps=1e-12)
coef = res[1].ravel()
elif method is "logcosh":
coef = logcosh_reg(X_train, Y_train, lambda_)
elif method is "linex":
coef = linex_reg(X_train, Y_train, lambda_)
# Ranking on the test
mu = X_test.dot(coef)
sorted_residual = np.sort(np.abs(Y_test - mu))
index = int((X.shape[0] / 2 + 1) * (1 - alpha))
quantile = sorted_residual[index]
mu = X[-1, :].dot(coef)
return intervals.closed(mu - quantile, mu + quantile)
def LassoPath(self,
alpha,
eps=1e-2): # eps the smaller it is the longer is the path
sel_log_LassoAlpha = -log10(alpha)
alphas_lasso, coefs_lasso, _ = lasso_path(self.X,
self.y,
eps,
fit_intercept=False)
colors = cycle(['b', 'r', 'g', 'c', 'k'])
neg_log_LassoAlphas = -log10(alphas_lasso)
fig, ax = plt.subplots()
for coef_l, c in zip(coefs_lasso, colors):
ax.plot(neg_log_LassoAlphas, coef_l, c=c)
ax.axvline(sel_log_LassoAlpha,
linestyle='--',
color='k',
label='$\\alpha$: CV estimate')
xunit, yunit = plot_unit([ax.get_xlim(), ax.get_ylim()])
ax.text(sel_log_LassoAlpha - xunit,
ax.get_ylim()[1] + yunit, round(sel_log_LassoAlpha, 2))
ax.set_xlabel(
'-log($\\alpha$)') # or # ax.set_xlabel(r'-log($\alpha$)')
ax.set_ylabel('Coefficients')
ax.set_title('Lasso Path')
fig.savefig(
os.path.join(self.fpath, '_'.join([self.name, 'X_LassoPath.png'])))
def _gen_cv_paths(self, alphas):
"""Helper function to generate lasso paths."""
self.alphas, self.coefs_cv, _ = linear_model.lasso_path(
self.x_train,
self.y_train,
fit_intercept=self.intercept,
alphas=alphas)
self.coefs_cv = self.coefs_cv.T
def cross_val(X, y, lambdas, sigma_0, eps=1e-4, method="lasso", KF=None):
"""
Perform a 5-fold cross-validation and return the mean square errors for
different parameters lambdas.
"""
n_samples, n_features = X.shape
n_lambdas = len(lambdas)
if KF is None:
KF = KFold(n_samples, 5, shuffle=True, random_state=42)
n_folds = KF.n_folds
errors = np.zeros((n_lambdas, n_folds))
i_fold = 0
for train_index, test_index in KF:
X_train = X[train_index]
X_test = X[test_index]
y_train = y[train_index]
y_test = y[test_index]
if method == "smoothed_concomitant":
betas, sigmas, gaps, n_iters, _ = \
SC_path(X_train, y_train, lambdas, eps=eps,
sigma_0=sigma_0)
elif method == "lasso":
betas = lasso_path(X_train, y_train, alphas=lambdas, tol=eps)[1]
betas = betas.T
elif method in ["ls_smoothed_concomitant", "ls_lasso"]:
betas = estimator_LS(X_train,
y_train,
lambdas,
sigma_0=sigma_0,
eps=eps,
method=method)
elif method == "SZ_path":
betas, sigmas = SZ_path(X_train, y_train, eps, lambdas)
elif method == "SBvG":
betas, sigmas = SBvG_path(X_train, y_train, lambdas)
elif method == "belloni":
betas, sigmas = belloni_path(X_train, y_train, lambdas)
else:
1 / 0 # BOOM !
for l in range(n_lambdas):
y_pred = np.dot(X_test, betas[l, :])
errors[l, i_fold] = np.mean((y_pred - y_test)**2)
i_fold += 1
return np.mean(errors, axis=1)
def _alpha_grid(X, y, center, XTX):
n, p = X.shape
alphas, coefs, _ = lasso_path(X.copy(),
y.copy(),
Xy=center.copy(),
precompute=XTX.copy())
nselected = np.count_nonzero(coefs, axis=0)
alphas = alphas[nselected < 20]
return alphas
def fit_lfm_lasso_path(x, z, p=None, lambdas=None, fit_intercept=True):
"""For details, see here.
Parameters
----------
x : array, shape (t_, n_)
z : array, shape (t_, k_)
p : array, optional, shape (t_,)
lambdas : array, shape(i_,), optional
fit_intercept : bool, optional
Returns
-------
alpha : array, shape (i_, n_)
beta : array, shape (i_, n_, k_)
"""
if len(x.shape) == 1:
x = x.reshape(-1, 1)
if len(z.shape) == 1:
z = z.reshape(-1, 1)
t_, n_ = x.shape
k_ = z.shape[1]
if lambdas is None:
lambdas = np.array([0, 0.1, 0.2])
i_ = lambdas.shape[0]
if p is None:
p = np.ones(t_) / t_
if fit_intercept is True:
m_x = p @ x
m_z = p @ z
else:
m_x = np.zeros(n_, )
m_z = np.zeros(k_, )
x_p = ((x - m_x).T * np.sqrt(p)).T
z_p = ((z - m_z).T * np.sqrt(p)).T
_, coeff_, _ = lasso_path(z_p,
x_p,
alphas=lambdas / (2 * t_),
fit_intercept=False)
# lasso_path automatically sorts lambdas from the largest to the smallest,
# so we have to revert order of coeff_ back to the original lambdas
idx = np.argsort(lambdas)[::-1]
betas = np.zeros((i_, n_, k_))
betas[idx, :, :] = coeff_.transpose((2, 0, 1))
alphas = m_x - betas @ m_z
return alphas, betas
def plot_process(self,
eps=5e-3,
title="Lasso coef Plot",
save=False,
file_path=None):
'''
Ploting the process of finding the best features
:param eps: float
Length of the path
:param title string
The title of the plot
:param save boolean, default = False
If the this parameter is set to False that the model will not save the model
If it is set to True the plot will be saved using :param file_path
:param file_path: string, default = None
The file path where the plot will be saved
If the :param save is set to False the it is not used
:return:
Plots the process of the algorithm
'''
X = self.dataframe[self.X_columns].values
y = self.dataframe[self.y_column].values
alphas = np.linspace(self.lasso_cv.alpha_ - 0.1,
self.lasso_cv.alpha_ + 0.1,
self.n_alphas,
endpoint=True)
alphas_lasso, coefs_lasso, _ = lasso_path(X,
y,
eps,
fit_intercept=False,
alphas=alphas)
neg_log_alphas_lasso = alphas_lasso
max_coef = coefs_lasso[0][0]
min_coef = coefs_lasso[0][0]
for i in range(len(coefs_lasso)):
line_style = lambda col: '-' if col in self.choosed_cols else '--'
plt.plot(neg_log_alphas_lasso,
coefs_lasso[i],
line_style(self.X_columns[i]),
label=self.X_columns[i])
if max(coefs_lasso[i]) > max_coef:
max_coef = max(coefs_lasso[i])
if min(coefs_lasso[i]) < min_coef:
min_coef = min(coefs_lasso[i])
plt.vlines(self.lasso_cv.alpha_,
min_coef,
max_coef,
linestyles='dashed')
plt.xlabel('-Log(alpha)')
plt.ylabel('coefficients')
plt.title(title)
plt.axis('tight')
plt.legend()
if save:
plt.savefig(file_path)
plt.show()
def test_compute_alo_path_fast(method):
X, y = make_test_case(50, 20, 10)
alphas, beta_hats, _ = linear_model.lasso_path(X, y)
alo = lasso.compute_alo_lasso_reference(X, y, beta_hats)
alo_fast = method(X, y, beta_hats)
assert np.all(np.isfinite(alo) == np.isfinite(alo_fast))
assert np.square(alo[np.isfinite(alo)] -
alo_fast[np.isfinite(alo_fast)]).mean() < 1e-3
def path_calc(X, y, X_holdout, y_holdout, alphas, paramgrid, colname = 'CV', yname = '', method = 'Elastic Net'):
#make a copy of the parameters before popping things off
copy_params = copy.deepcopy(paramgrid)
fit_intercept = copy_params.pop('fit_intercept')
precompute = copy_params.pop('precompute')
copy_X = copy_params.pop('copy_X')
normalize = False
# this code adapted from sklearn ElasticNet fit function, which unfortunately doesn't accept multiple alphas at once
X, y = check_X_y(X, y, accept_sparse='csc',
order='F', dtype=[np.float64, np.float32],
copy=copy_X and fit_intercept,
multi_output=True, y_numeric=True)
y = check_array(y, order='F', copy=False, dtype=X.dtype.type,
ensure_2d=False)
#this is the step that gives the data to find intercept if fit_intercept is true.
X, y, X_offset, y_offset, X_scale, precompute, Xy = _pre_fit(X, y, None, precompute, normalize,
fit_intercept, copy=False)
y = np.squeeze(y)
#do the path calculation, and tell how long it took
print('Calculating path...')
start_t = time.time()
if method == 'Elastic Net':
path_alphas, path_coefs, path_gaps, path_iters = enet_path(X, y, alphas=alphas, return_n_iter = True,
**copy_params)
if method == 'LASSO':
path_alphas, path_coefs, path_gaps, path_iters = lasso_path(X, y, alphas=alphas, return_n_iter=True,
**copy_params)
dt = time.time() - start_t
print('Took ' + str(dt) + ' seconds')
#create some empty arrays to store the result
y_pred_holdouts = np.empty(shape=(len(alphas),len(y_holdout)))
intercepts = np.empty(shape=(len(alphas)))
rmses = np.empty(shape=(len(alphas)))
cvcols = []
for j in list(range(len(path_alphas))):
coef_temp = path_coefs[:, j]
if fit_intercept:
coef_temp = coef_temp / X_scale
intercept = y_offset - np.dot(X_offset, coef_temp.T)
else:
intercept = 0.
y_pred_holdouts[j,:] = np.dot(X_holdout, path_coefs[:, j]) + intercept
intercepts[j] = intercept
rmses[j] = RMSE(y_pred_holdouts[j,:], y_holdout)
cvcols.append(('predict','"'+ method + ' - ' + yname + ' - ' + colname + ' - Alpha:' + str(path_alphas[j]) + ' - ' + str(paramgrid) + '"'))
return path_alphas, path_coefs, intercepts, path_iters, y_pred_holdouts, rmses, cvcols
def test_mtl_path():
X, Y = build_dataset(n_targets=3)
tol = 1e-10
params = dict(eps=0.01, tol=tol, n_alphas=10)
alphas, coefs, gaps = mtl_path(X, Y, **params)
np.testing.assert_array_less(gaps, tol)
sk_alphas, sk_coefs, sk_gaps = lasso_path(X, Y, **params, max_iter=10000)
np.testing.assert_array_less(sk_gaps, tol * np.linalg.norm(Y, 'fro')**2)
np.testing.assert_array_almost_equal(coefs, sk_coefs, decimal=5)
np.testing.assert_allclose(alphas, sk_alphas)
def test_mtl():
# n_samples, n_features = 30, 70
# X, Y, _, _ = build_dataset(n_samples, n_features, n_targets=10)
X, Y, _, _ = build_dataset(n_targets=10)
tol = 1e-9
alphas, coefs, gaps = mtl_path(X, Y, eps=1e-2, tol=tol)
np.testing.assert_array_less(gaps, tol)
sk_alphas, sk_coefs, sk_gaps = lasso_path(X, Y, eps=1e-2, tol=tol)
np.testing.assert_array_less(sk_gaps, tol * np.linalg.norm(Y, 'fro')**2)
np.testing.assert_array_almost_equal(coefs, sk_coefs, decimal=5)
np.testing.assert_allclose(alphas, sk_alphas)
def Plot_Lasso_Path(self, path_lenght=5e-3, alphas=None):
import matplotlib.pyplot as plt
print("Computing regularization path using the lasso...")
alphas_lasso, coefs_lasso, _ = lasso_path(
self.X, self.Y, path_lenght, fit_intercept=False, alphas=alphas)
print("Computing regularization path using the positive lasso...")
alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path(
self.X, self.Y, path_lenght, positive=True, fit_intercept=False, alphas=alphas)
plt.figure()
ax = plt.gca()
ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k'])
l1 = plt.plot(-np.log10(alphas_lasso), coefs_lasso.T)
l2 = plt.plot(-np.log10(alphas_positive_lasso), coefs_positive_lasso.T,
linestyle='--')
plt.xlabel('-Log(alpha)')
plt.ylabel('coefficients')
plt.title('Lasso and positive Lasso')
plt.legend(
(l1[-1], l2[-1]), ('Lasso', 'positive Lasso'), loc='lower left')
plt.axis('tight')
plt.show()
def test_lasso_path(self):
diabetes = datasets.load_diabetes()
df = pdml.ModelFrame(diabetes)
result = df.linear_model.lasso_path()
expected = lm.lasso_path(diabetes.data, diabetes.target)
self.assertEqual(len(result), 3)
tm.assert_numpy_array_equal(result[0], expected[0])
self.assertIsInstance(result[1], pdml.ModelFrame)
tm.assert_index_equal(result[1].index, df.data.columns)
self.assert_numpy_array_almost_equal(result[1].values, expected[1])
self.assert_numpy_array_almost_equal(result[2], expected[2])
result = df.linear_model.lasso_path(return_models=True)
expected = lm.lasso_path(diabetes.data, diabetes.target, return_models=True)
self.assertEqual(len(result), len(expected))
self.assertIsInstance(result, tuple)
tm.assert_numpy_array_equal(result[0], result[0])
tm.assert_numpy_array_equal(result[1], result[1])
tm.assert_numpy_array_equal(result[2], result[2])
def train(self, features, labels):
from sklearn import linear_model
betas = []
xs = []
for ci,ells in enumerate(labels.T):
active = ~np.isnan(ells)
fi = features[active]
ells = ells[active]
fits = linear_model.lasso_path(fi, ells)
xs.append(fits[-1].coef_.T.copy())
betas.append(fits[-1].intercept_.copy())
return product_intercept_predictor(np.array(xs).T, np.array(betas))
def test_celer_path_vs_lasso_path(sparse_X, prune):
"""Test that celer_path matches sklearn lasso_path."""
X, y, _, _ = build_dataset(n_samples=30, n_features=50, sparse_X=sparse_X)
params = dict(eps=1e-2, n_alphas=10, tol=1e-14)
alphas1, coefs1, gaps1 = celer_path(
X, y, return_thetas=False, verbose=1, prune=prune, **params)
alphas2, coefs2, gaps2 = lasso_path(X, y, verbose=False, **params)
np.testing.assert_allclose(alphas1, alphas2)
np.testing.assert_allclose(coefs1, coefs2, rtol=1e-05, atol=1e-6)
def test_lasso_path(self):
diabetes = datasets.load_diabetes()
df = pdml.ModelFrame(diabetes)
result = df.linear_model.lasso_path()
expected = lm.lasso_path(diabetes.data, diabetes.target)
self.assertEqual(len(result), 3)
tm.assert_numpy_array_equal(result[0], expected[0])
self.assertIsInstance(result[1], pdml.ModelFrame)
tm.assert_index_equal(result[1].index, df.data.columns)
self.assert_numpy_array_almost_equal(result[1].values, expected[1])
self.assert_numpy_array_almost_equal(result[2], expected[2])
result = df.linear_model.lasso_path(return_models=True)
expected = lm.lasso_path(diabetes.data, diabetes.target, return_models=True)
self.assertEqual(len(result), len(expected))
self.assertIsInstance(result, tuple)
tm.assert_numpy_array_equal(result[0], result[0])
tm.assert_numpy_array_equal(result[1], result[1])
tm.assert_numpy_array_equal(result[2], result[2])
def do_lasso_simulation(data, NUM_LAMBDA_SPLITS=3000):
# Make lasso path
lasso_path, coefs, _ = linear_model.lasso_path(
data.X_train,
np.array(data.y_train.flatten().tolist()[0]), # reshape appropriately
method='lasso'
)
prob = LassoProblemWrapper(
data.X_train,
data.y_train
)
# print "lasso_path", lasso_path
val_errors = []
for i, l in enumerate(lasso_path):
beta = prob.solve(np.array([l]))
val_error = testerror_lasso(data.X_validate, data.y_validate, beta)
val_errors.append(val_error)
sorted_idx = np.argsort(val_errors)
max_lam = lasso_path[np.min(sorted_idx[:3])]
min_lam = lasso_path[np.max(sorted_idx[:3])]
print "min_lam", min_lam, "max_lam", max_lam
print "lasso_path[sorted_idx[:3]]", lasso_path[sorted_idx[:3]]
finer_lam_range = []
for i, l_idx in enumerate(range(np.min(sorted_idx[:3]) - 1, np.max(sorted_idx[:3]) + 1)):
fudge = 0
if i == 0:
fudge = 1e-10
l_min = lasso_path[l_idx + 1] if lasso_path.size - 1 >= l_idx + 1 else 0
l_max = lasso_path[l_idx] if l_idx >= 0 else lasso_path[0] + 0.1
add_l = np.arange(start=l_min, stop=l_max + fudge, step=(l_max - l_min)/NUM_LAMBDA_SPLITS)
finer_lam_range.append(add_l)
finer_lam_range = np.concatenate(finer_lam_range)
print "finer_lam_range min", np.min(finer_lam_range), "max", np.max(finer_lam_range)
fine_val_errors = []
for i, l in enumerate(finer_lam_range):
beta = prob.solve(np.array([l]))
val_error = testerror_lasso(data.X_validate, data.y_validate, beta)
fine_val_errors.append(val_error)
fine_sorted_idx = np.argsort(fine_val_errors)
best_lam = finer_lam_range[fine_sorted_idx[0]]
print "best_lam", best_lam
min_dist, idx = get_dist_of_closest_lambda(best_lam, lasso_path)
print "min_dist", min_dist
return min_dist
def test_compute_alo_path():
X, y = make_test_case(50, 20, 10)
alphas, beta_hats, _ = linear_model.lasso_path(X, y)
alo = lasso.compute_alo_lasso_reference(X, y, beta_hats)
beta_hat = beta_hats[:, 5]
h_5 = lasso.compute_h_lasso(X, np.abs(beta_hat) > 1e-5)
r_5 = y - np.dot(X, beta_hat)
alo_5 = np.square(r_5 / (1 - h_5)).mean()
assert len(alo) == len(alphas)
assert np.allclose(alo_5, alo[5])
def get_coef_range(X, y):
print "starting experiment"
#with stopwatch("lasso paths"):
alphas, coefs, dual_gaps = lasso_path(
X,
y,
#l1_ratio=1.0,
verbose=True,
#return_models=False,
positive=False,
max_iter=1000)
return alphas, coefs, dual_gaps
def my_lasso_path(U, F):
alphas = np.logspace(-10, -3, num=50)
print 'alphas'
print alphas
a, b, c = lasso_path(U,
F,
alphas=alphas,
fit_intercept=False,
normalize=False,
max_iter=1000,
tol=0.000001,
selection='random')
print 'alpha n_nonzero'
t = np.sum(b != 0, 0)
for aa, tt in zip(a.tolist(), t.tolist()):
print str(aa) + '\t' + str(tt)
print
a_st, b_st, c_st = lasso_path(U,
F,
alphas=alphas,
fit_intercept=True,
normalize=True,
max_iter=1000,
tol=0.000001,
selection='random')
print 'alpha n_nonzero'
t_st = np.sum(b_st != 0, 0)
for aa, tt, tt_st in zip(a.tolist(), t.tolist(), t_st.tolist()):
print[aa, tt, tt_st]
print
return a, b, c
def test_compute_alo_leverage():
X, y = make_test_case(50, 20, 10)
alphas, beta_hats, _ = linear_model.lasso_path(X, y)
alo_fast, leverage = native_impl.lasso_compute_alo(X,
y,
beta_hats,
return_leverage=True)
assert alo_fast.shape == (beta_hats.shape[1], )
assert leverage.shape == (50, beta_hats.shape[1])
assert np.all(leverage >= 0)
assert np.all(leverage <= 1)
def plot_lasso_path(df, resp_var, exp_vars):
""" Plot the lasso path. Both response and explanatory
variables are standardised first.
Args:
df: Dataframe
resp_var: String. Response variable
exp_vars: List of strings. Explanatory variables
Returns:
Dataframe of path and matplotlib figure with tooltip-
labelled lines. To view this figure in a notebook, use
mpld3.display(f) on the returned figure object, f.
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import mpld3
from mpld3 import plugins
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import lasso_path
# Standardise the feature data and response
feat_std = StandardScaler().fit_transform(df[[resp_var,] + exp_vars])
# Calculate lasso path
alphas, coefs, _ = lasso_path(feat_std[:, 1:], # X
feat_std[:, 0], # y
eps=1e-3, # Path length
fit_intercept=False) # Already centred
# -Log(alphas) is easier for display
neg_log_alphas = -np.log10(alphas)
# Build df of results
res_df = pd.DataFrame(data=coefs.T, index=alphas, columns=exp_vars)
# Plot
fig, ax = plt.subplots()
for coef, name in zip(coefs, exp_vars):
line = ax.plot(neg_log_alphas, coef, label=name)
plugins.connect(fig, plugins.LineLabelTooltip(line[0], label=name))
plt.xlabel('-Log(alpha)')
plt.ylabel('Coefficients')
plt.title('Lasso paths')
plt.legend(loc='best', title='', ncol=3)
return res_df, fig
def L_U(X, y, eps=1e-4, max_iter=5000):
n_samples, n_features = X.shape
# Lasso with universal lambda
univ_lambda = np.sqrt(2. * np.log(n_features) / float(n_samples))
beta_ulasso = lasso_path(X,
y,
alphas=[univ_lambda],
tol=eps,
max_iter=max_iter)[1]
beta_ulasso = beta_ulasso.ravel()
size = np.sum(beta_ulasso != 0)
sigma_ulasso = np.linalg.norm(y - np.dot(X, beta_ulasso))
sigma_ulasso /= np.sqrt(n_samples - size)
return beta_ulasso, sigma_ulasso
def solvePenalizedLSQ(A, b, eps, B, c):
"""
Solves an unconstrained linear least squares problem
minimize 2-norm (A*x-b)^2 + eps * 2-norm (Bx-c)^2 + tau 1-norm(x)
"""
X = numpy.vstack((A, eps * B))
y = numpy.concatenate((b, c))
print "Computing regularization path using the lasso..."
models = lasso_path(X, y)
alphas_lasso = numpy.array([model.alpha for model in models])
coefs_lasso = numpy.array([model.coef_ for model in models])
print alphas_lasso
print coefs_lasso
def fit(self, X, y, feature_labels, estimator_params=None):
"""Computes the Lasso path using Sklearn's lasso_path method.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data (the independent variables).
y : array-like, shape (n_samples, n_outputs)
Training data (the output/target values).
feature_labels : array-like, shape (n_features)
Labels for each of the features in X.
estimator_params : dict, optional
The parameters to pass to Sklearn's Lasso estimator.
Returns
-------
self
"""
self._reset()
if estimator_params is None:
estimator_params = {}
self.feature_labels_ = feature_labels
alphas, coefs, _ = lasso_path(X, y, **estimator_params)
self.alphas_ = alphas.copy()
self.coefs_ = coefs.copy()
# Rank the features in X by order of importance. This ranking is based
# on how early a given features enter the regression (the earlier a
# feature enters the regression, the MORE important it is).
feature_rankings = [[] for _ in range(X.shape[1])]
for target_coef_paths in self.coefs_:
for i, feature_path in enumerate(target_coef_paths):
entrance_step = 1
for val_at_step in feature_path:
if val_at_step == 0:
entrance_step += 1
else:
break
feature_rankings[i].append(entrance_step)
self.rankings_ = np.array([np.mean(ranks) for ranks in feature_rankings])
return self
from scipy.io import mmwrite, mmread
from sklearn.linear_model.coordinate_descent import _alpha_grid
from sklearn.linear_model import lasso_path
from
from time import time
# Newsgroup datasets
X_new = mmread("X_new.mtx")
y_new = mmread("y_new.mtx")
y_new = y_new.toarray()[0]
#alphas = _alpha_grid(X_new, y_new, eps=1e-3, fit_intercept=True, normalize=False, n_alphas=100)
t = time()
lasso_path(X_new, y_new, eps=1e-3, precompute=False, fit_intercept=True, normalize=False, n_alphas=100)
print time() - t
X_new = mmread("haxby_X.mtx").toarray()
y_new = mmread("haxby_y.mtx").toarray()[0]
#print y_new.shape
#y_new = y_new.toarray()[0]
# alphas = _alpha_grid(X_new, y_new, eps=1e-3, fit_intercept=True, normalize=False, n_alphas=100)
t = time()
coef = lasso_path(X_new, y_new, eps=1e-3, precompute=False, fit_intercept=True, normalize=False, alphas=alphas)#n_alphas=100)
coef_ = coef[1]
sq_loss = np.sum(0.5*(y_new[:, np.newaxis] - safe_sparse_dot(X_new, coef_))**2, axis=0)
l2_penalty = np.sum(coef_**2, axis=0)
l1_penalty = np.sum(np.abs(coef_), axis=0)
print time() - t
def source(self, numpoints=100, L=0.5, por=0.0, LocR=None, srcTime=None):
event = self.event
# precondiciones
assert(0 <= por)
assert(L > 0)
assert(numpoints == int(numpoints))
if LocR is None:
LocX, LocY, LocZ = (event.LocX, event.LocY, event.LocZ)
else:
LocX, LocY, LocZ = LocR
if srcTime is None:
self.srcTime = dateTime2Num(event.origin_time) + linspace(-por * L, (1 - por) * L, numpoints)
'''
reconstruccion de la fuente
:param event: objeto del tipo event
:param numpoints=100: numero de la discretizacion de la fuente estimada
:param L: Largo de la ventana de tiempo de la fuente entimada
:param por: fraccion de tiempo antes del tiempo estimado por Codelco
'''
dt = self.srcTime[1] - self.srcTime[0]
"""
se requiere resolver un sistema lineal del tipo A*alphas = U
"""
A, U = ([], [])
# agregar el campo de desplazamiento a el vector de respuesta
# los id son los mismos!
for gs in event.seismograms:
# se agregan todas las dimensiones que mantienen mediciones validas
data = gs.data.values
if gs.X_enabled == 1:
U = hstack((U, data[:, 0].T))
if gs.Y_enabled == 1:
U = hstack((U, data[:, 1].T))
if gs.Z_enabled == 1:
U = hstack((U, data[:, 2].T))
for G in event.seismograms:
# frecuencia de muestreo
hsr = G.hardware_sampling_rate
# la relacion dt*hsr > 1
deltat = dt * hsr
#assert dt * hsr <= 1 , 'Advertencia: el producto dt * hsr deberia ser mayor que 1'
R = (G.x_coord - LocX, G.y_coord - LocY, G.z_coord - LocZ)
# funcion de green
# t = G.timevector - dateTime2Num(date=event.origin_time)
t = G.timevector - self.srcTime[0]
alpha = G.P_velocity
beta = G.S_velocity
rho = G.RockDensity
Gk = GreenKernel(R=R, time=t, alpha=alpha, beta=beta, rho=rho)
assert(not any(isnan(Gk[:])))
# integracion de la funcion de Green
dtdomain = t[1] - t[0]
F = cumsum(Gk, axis=2) * dtdomain
FF = zeros(shape(F))
# matriz auxiliar en donde se almacenaran todas las convoluciones
# producidas en un solo sensor.
B = []
for jj in xrange(numpoints):
# para todo elemento de la base
ii = xrange(size(F, 2))
# indices para los saltos en la convolucion entre la base y la
# funcion de Green
tf = map(lambda I: int(max(I - floor(jj * deltat), 0)), ii)
ti = map(lambda I: int(max(I - floor((jj + 1) * deltat), 0)), ii)
# convolucion con respecto la base seleccionada
FF[:, :, ii] = F[:, :, tf] - F[:, :, ti]
# convolucion para un elemento de la base
C = []
if G.X_enabled:
if C == []:
C = FF[0, :, :].copy()
else:
C = hstack((C, FF[0, :, :].copy()))
assert(not any(isnan(C[:])))
if G.Y_enabled:
if C == []:
C = FF[1, :, :].copy()
else:
C = hstack((C, FF[1, :, :].copy()))
assert(not any(isnan(C[:])))
if G.Z_enabled:
if C == []:
C = FF[2, :, :].copy()
else:
C = hstack((C, FF[2, :, :].copy()))
assert(not any(isnan(C[:])))
if B == []:
B = C.copy()
else:
B = vstack((B, C.copy()))
if A == []:
A = B.copy()
else:
A = hstack((A, B.copy()))
# @todo: minimizar la norma 1 para hacer la estimacion mas robusta
# resolucion del sistema lineal que minimiza la suma de la norma 2 de error
assert(not any(isnan(A[:])))
#from scipy.sparse import csr_matrix
#from scipy.sparse.linalg import lsqr
#matrix = csr_matrix(A.T)
# X = numpy.linalg.lstsq(A.T, U)[0]
# regresion lineal
lasso = None
if lasso == True:
_, lasso_path, _ = lasso_path(A.T, U)
rgr_lasso = Lasso()
rgr_lasso.fit(A.T, U)
X = rgr_lasso.coef_
pass
if det(dot(A, A.T)) != 0:
#invertible
X = dot(dot(U, A.T), inv(dot(A, A.T)))
else:
#no invertible
X = dot(dot(U, A.T), pinv(dot(A, A.T)))
src = zip(self.srcTime.T,
X[range(0, 3 * numpoints, 3)].T,
X[range(1, 3 * numpoints, 3)].T,
X[range(2, 3 * numpoints, 3)].T
)
# post condiciones
assert(shape(src) == (numpoints, 4))
# error de estimacion
error = norm(U - dot(X, A), 2)
src = array(src)
rot, vec, val = _rotate(src)
#condiciones necesarias de orden de los valores propios y las coordenadas de
#la fuente
order = sorted(range(3), key=lambda k:val[k])
val = val[order]
#assert(val[0] <= val[1] <= val[2])
rot[:, 1:4] = rot[:, 1:4][:, order]
return(src, error, rot, vec, val)
plt.subplot(121)
m_log_alphas = -np.log10(lasso.steps[1][1].alphas_)
plt.plot(m_log_alphas, lasso.steps[1][1].mse_path_, ':')
plt.plot(m_log_alphas, lasso.steps[1][1].mse_path_.mean(axis=-1), 'k',
label='Average across the folds', linewidth=2)
plt.axvline(-np.log10(lasso.steps[1][1].alpha_), linestyle='--', color='k',
label='alpha: CV estimate')
plt.legend(loc = 'upper left')
plt.xlabel('-Log(alpha)')
plt.ylabel('Mean square error')
plt.title('Mean square error on each fold')
plt.axis('tight')
# Plot lasso paths
plt.subplot(122)
alphas_lasso, coefs_lasso, _ = lasso_path(features, labels, alphas = alphas)
plt.plot(-np.log10(alphas_lasso), coefs_lasso.T)
plt.axvline(-np.log10(lasso.steps[1][1].alpha_), linestyle='--', color='k',
label='alpha: CV estimate')
plt.xlabel('-Log(alpha)')
plt.ylabel('Coefficients')
plt.title('Lasso Paths')
plt.axis('tight')
lasso_coef = pd.DataFrame({'coef': lasso.steps[1][1].coef_.tolist()}, features_list[1:]).round(3)
lasso_selection = lasso_coef[lasso_coef['coef']!=0]
lasso_selection
print "Lasso selected an alpha of %.2f with %d features:" % (alpha_lasso, len(lasso_selection))
lasso_selection.sort('coef', ascending=False)
def lasso_coefs(X, y):
_, coefs, _ = lasso_path(X, y)
return coefs.T
def main():
path = '/users/davecwright/documents/kaggle/liberty_fire_cost/'
path = 'c:\\users\\dwright\\code\\'
train_name = path + 'train.csv'
test_name = path + 'test.csv'
# f = open(path + 'train.csv', 'rb')
readRows = 2000 #None for all
print 'loading train_data'
train_data = pd.read_csv(train_name, nrows=readRows)
print 'train_data loaded'
y_data = train_data['target'].values
train_data.drop('target', 1)
train_data = scrub(train_data)
y_data = y_data.astype(float)
eps = 5e-3
X = train_data
y = y_data
folds = 10
kf = cross_validation.KFold(y_data.shape[0], n_folds=folds)
alphas = range(folds)
k = 0
for test, train in kf:
penalty = (alphas[k] + 1) * 1/folds
print alpha
clf = ElasticNet(l1_ratio=penalty, eps=eps)
clf.train(X, y)
# doing our own cv parametarization
print 'computing lasso path'
alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept = False)
print 'computing enet path'
alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False)
plt.figure(1)
ax = plt.gca()
ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k'])
l1 = plt.plot(-np.log10(alphas_lasso), coefs_lasso.T)
l2 = plt.plot(-np.log10(alphas_enet), coefs_enet.T, linestyle='--')
plt.xlabel('-Log(alpha)')
plt.ylabel('coefficients')
plt.title('Lasso and Elastic-Net Paths')
plt.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left')
plt.axis('tight')
plt.show()
sys.exit()
for train_index, test_index in kf:
#print train_index, test_index
print train_data.iloc[train_index], train_data.iloc[test_index]
# I am implementing the lasso reduction here...
#for train, test in kf:
fig = plt.figure(figsize=(12, 9))
#ax = fig.add_subplot(111)
#ax.plot(np.sort(y_data))
#plt.xlabel("Number of Features")
#plt.ylabel("claims cost")
#plt.title("claims_cost")
#ax.set_xscale("log")
#ax.set_position([box.x0, box.y0 + box.height * 0.3, box.width, box.height * 0.7])
#ax.legend(**_PLT_LEGEND_OPTIONS)
#plt.show()
train_data = train_data.drop('target', 1)
# A - preprocessing
# encode the text variables, var1-var9, Z values are NaN
# fill in missing values in the text variables and in the continuous variables
# skip continuous for now
# A3. build new features through the interactions of various items
# skip for now
# A4. dimensionality reduction to take the feature set back down to something more manageable.
est_clf = svm.SVR(kernel='linear', C=1)
rks = select_ests(X_train, y_data, 100, est_clf)
X_train = X_train[:, rks]
clf = svm.SVR(kernel='linear', C=1)
acy = cv(X_train, y_data, clf, None, estimator_name(clf))
# the point here is to understand what accuracy is
print 'accuracy:', acy
#select_model(X_train, y_data)
# B. split out test and fit sets
test_data = pd.read_csv(test_name, nrows=readRows)
test_data = encode_impute(test_data)
X_test = scaler.transform(test_data)
for alpha in alphas:
simpLasCoefs, Yhat = simpleLasso(X,y,alpha,True)
# print "alpha = ", alpha
# print simpLasCoefs
# print Yhat
simpLasCoefsCR, YhatCR = simpleLasso(X_cr,y_cr,alpha,False)
# print simpLasCoefsCR
# print YhatCR
################------ Exercice 2.4 ------###############################
print "Ex. 2.4: Figure"
#_, theta_lasso, _ =lasso_path(np.array(X), np.array(y), alphas=alphas, fit_intercept=True, return_models=False)
_, theta_lasso_CR, _ =lasso_path(np.array(X_cr), np.array(y_cr), alphas=alphas, fit_intercept=False, return_models=False)
# plot lasso path
# fig1=plt.figure(figsize=(12,8))
# plt.title("Chemin du Lasso: "+ r"$p={0}, n={1} $".format(nfeatures,nsamples),fontsize = 16)
# ax1 = fig1.add_subplot(111)
# ax1.plot(alphas,np.transpose(theta_lasso),linewidth=3)
# ax1.set_xscale('log')
# ax1.set_xlabel(r"$\lambda$")
# ax1.set_ylabel("Amplitude des coefficients")
# ax1.set_ylim([-2,0.5])
# ax1.set_xlim([lstart,lend])
# plt.show(block=False)
fig2=plt.figure(figsize=(12,8))
plt.title("Chemin du Lasso vars_CR: "+ r"$p={0}, n={1} $".format(nfeatures,nsamples),fontsize = 16)
#Convert list of list to np array for input to sklearn packages
#Unnormalized labels
Y = numpy.array(labels)
#normalized lables
Y = numpy.array(labelNormalized)
#Unnormalized X's
X = numpy.array(xList)
#Normlized Xss
X = numpy.array(xNormalized)
alphas, coefs, _ = linear_model.lasso_path(X, Y, return_models=False)
plot.plot(alphas,coefs.T)
plot.xlabel('alpha')
plot.ylabel('Coefficients')
plot.axis('tight')
plot.semilogx()
ax = plot.gca()
ax.invert_xaxis()
plot.show()
nattr, nalpha = coefs.shape
#find coefficient ordering
data = load_mnist() pos_ind = 6 neg_ind = 5 sig_D = 100 # lmda_list = [0.0005, 0.001, 0.01, 0.1, 0.3] x, y = convert_binary(data, pos_ind, neg_ind) n, p = x.shape x = x.astype(float) y = y.astype(float) min_max_scaler = preprocessing.MinMaxScaler(feature_range=(0, 1)) x = min_max_scaler.fit_transform(x) # xtest = min_max_scaler.transform(x) # ntrain = ytrain.size alphas, coefs, gaps = linear_model.lasso_path(x, y, n_alphas=5, return_models=False, fit_intercept=False) lmda_list = alphas[::-1] n_iter = 10 ss = cv.StratifiedShuffleSplit(y=y, n_iter=n_iter, test_size=0.3, random_state=5) nzs_scg_T = np.zeros((n_iter, len(lmda_list))) nzs_scg_bar = np.zeros((n_iter, len(lmda_list))) nzs_rda_T = np.zeros((n_iter, len(lmda_list))) nzs_rda_bar = np.zeros((n_iter, len(lmda_list))) nzs_rda2_T = np.zeros((n_iter, len(lmda_list))) nzs_rda2_bar = np.zeros((n_iter, len(lmda_list))) nzs_cd_T = np.zeros((n_iter, len(lmda_list))) nzs_cd_bar = np.zeros((n_iter, len(lmda_list))) nzs_sgd = np.zeros((n_iter, len(lmda_list))) nsweep = 5 b = 5 c = 1
xsum = x.sum(axis=0)
# ind = np.where(xsum>0) # return object is tuple
# x = x[:, ind[0]]
x = x.astype(float)
y = y.astype(float)
n, p = x.shape
random_state = 21
lmda = 0.01
nsweep = 1
xtrain, xtest, ytrain, ytest = cv.train_test_split(x, y, test_size=0.2, random_state=random_state)
min_max_scaler = preprocessing.MinMaxScaler(feature_range=(0, 1))
xtrain = min_max_scaler.fit_transform(xtrain)
xtest = min_max_scaler.transform(xtest)
ntrain = ytrain.size
alphas, coefs, gaps = linear_model.lasso_path(xtrain, ytrain,n_alphas=10, return_models=False, fit_intercept=False)
alphas = alphas[::-1]
# zs = (coefs==0).sum(axis=0)
# zs = zs[::-1]
# gaps = gaps[::-1]
obj = np.zeros(len(alphas))
zs2 = np.zeros(len(alphas))
obj2 = np.zeros(len(alphas))
zs3 = np.zeros(len(alphas))
zs = np.zeros(len(alphas))
obj3 = np.zeros(len(alphas))
for i, alpha in enumerate(alphas):
print "alpha: %f" % alpha
print 'cd with random permutation'
clf = LassoLI(lmda=alpha, algo='cd', cd_ord='rand', T=6000)
from sklearn.linear_model import lasso_path, enet_path from sklearn import datasets diabetes = datasets.load_diabetes() X = diabetes.data y = diabetes.target X /= X.std(0) # Standardize data (easier to set the rho parameter) ################################################################################ # Compute paths eps = 5e-3 # the smaller it is the longer is the path print "Computing regularization path using the lasso..." models = lasso_path(X, y, eps=eps) alphas_lasso = np.array([model.alpha for model in models]) coefs_lasso = np.array([model.coef_ for model in models]) print "Computing regularization path using the elastic net..." models = enet_path(X, y, eps=eps, rho=0.8) alphas_enet = np.array([model.alpha for model in models]) coefs_enet = np.array([model.coef_ for model in models]) ################################################################################ # Display results ax = pl.gca() ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k']) l1 = pl.plot(coefs_lasso) l2 = pl.plot(coefs_enet, linestyle='--')
plt.axvline(BLLModel.alpha_, linestyle='--', label='CV Estimate of Best alpha')
plt.semilogx()
plt.legend()
ax = plt.gca()
ax.invert_xaxis()
plt.xlabel('alpha')
plt.ylabel('Mean Square Error')
plt.axis('tight')
plt.title('Determining alpha via LASSO 10-fold CV')
plt.show()
print "alpha Value that Minimizes CV Error ", BLLModel.alpha_
print "Minimum MSE ", min(BLLModel.mse_path_.mean(axis=-1))
bestAlpha = BLLModel.alpha_
alphas, coefs, _ = lasso_path(X, y, return_models=False)
plt.plot(alphas,coefs.T)
plt.xlabel('alpha')
plt.ylabel('Coefficients')
plt.axis('tight')
plt.title('Variables as they enter the model')
plt.semilogx()
plt.legend(loc='upper left')
ax = plt.gca()
ax.invert_xaxis()
plt.show()
nattr, nalpha = coefs.shape
#find coefficient ordering
nzList = []
print("Computing regularization path using the ridge...")
n_alphas = 200
alphas = np.logspace(-4, 2, n_alphas)
clf = linear_model.Ridge(fit_intercept=False)
coefs = []
for a in alphas:
clf.set_params(alpha=a, max_iter=1000)
clf.fit(xtrain, ytrain)
coefs.append(clf.coef_)
# lasso and elastic net path
eps = 5e-3 # the smaller it is the longer is the path
print("Computing regularization path using the lasso...")
alphas_lasso, coefs_lasso, _ = lasso_path(xtrain, ytrain, eps, fit_intercept=False)
print("Computing regularization path using the elastic net...")
alphas_enet, coefs_enet, _ = enet_path(
xtrain, ytrain, eps=eps, l1_ratio=0.8, fit_intercept=False)
# Display results
plt.figure(1)
ax = plt.gca()
ax.set_color_cycle(['b', 'r', 'g', 'c', 'k', 'y', 'm'])
ax.plot(alphas, coefs)
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1]) # reverse axis
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
X /= X.std(axis=0) # Standardize data (easier to set the l1_ratio parameter)
# Compute paths
eps = 5e-3 # the smaller it is the longer is the path
print("Computing regularization path using the lasso...")
# The return_models parameter sets that lasso_path will return
# the alphas and the coefficients as output, instead of a list
# of models as it does by default. Returning the list of models
# is deprecated and will eventually be removed in 0.15
alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, return_models=False)
print("Computing regularization path using the positive lasso...")
alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path(X, y, eps,
positive=True,
return_models=False)
print("Computing regularization path using the elastic net...")
alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=0.8,
return_models=False)
print("Computing regularization path using the positve elastic net...")
alphas_positive_enet, coefs_positive_enet, _ = enet_path(X, y, eps=eps,
l1_ratio=0.8,
positive=True,
return_models=False)
from sklearn.linear_model import lasso_path, enet_path
from sklearn import datasets
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
X /= X.std(0) # Standardize data (easier to set the l1_ratio parameter)
###############################################################################
# Compute paths
eps = 5e-3 # the smaller it is the longer is the path
print("Computing regularization path using the lasso...")
models = lasso_path(X, y, eps=eps)
alphas_lasso = np.array([model.alpha for model in models])
coefs_lasso = np.array([model.coef_ for model in models])
print("Computing regularization path using the positive lasso...")
models = lasso_path(X, y, eps=eps, positive=True)
alphas_positive_lasso = np.array([model.alpha for model in models])
coefs_positive_lasso = np.array([model.coef_ for model in models])
print("Computing regularization path using the elastic net...")
models = enet_path(X, y, eps=eps, l1_ratio=0.8)
alphas_enet = np.array([model.alpha for model in models])
coefs_enet = np.array([model.coef_ for model in models])
print("Computing regularization path using the positve elastic net...")
models = enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True)
def compute_bench(samples_range, features_range):
it = 0
results = defaultdict(lambda: [])
max_it = len(samples_range) * len(features_range)
for n_samples in samples_range:
for n_features in features_range:
it += 1
print('====================')
print('Iteration %03d of %03d' % (it, max_it))
print('====================')
dataset_kwargs = {
'n_samples': n_samples,
'n_features': n_features,
'n_informative': n_features // 10,
'effective_rank': min(n_samples, n_features) / 10,
#'effective_rank': None,
'bias': 0.0,
}
print("n_samples: %d" % n_samples)
print("n_features: %d" % n_features)
X, y = make_regression(**dataset_kwargs)
gc.collect()
print("benchmarking lars_path (with Gram):", end='')
sys.stdout.flush()
tstart = time()
G = np.dot(X.T, X) # precomputed Gram matrix
Xy = np.dot(X.T, y)
lars_path(X, y, Xy=Xy, Gram=G, method='lasso')
delta = time() - tstart
print("%0.3fs" % delta)
results['lars_path (with Gram)'].append(delta)
gc.collect()
print("benchmarking lars_path (without Gram):", end='')
sys.stdout.flush()
tstart = time()
lars_path(X, y, method='lasso')
delta = time() - tstart
print("%0.3fs" % delta)
results['lars_path (without Gram)'].append(delta)
gc.collect()
print("benchmarking lasso_path (with Gram):", end='')
sys.stdout.flush()
tstart = time()
lasso_path(X, y, precompute=True)
delta = time() - tstart
print("%0.3fs" % delta)
results['lasso_path (with Gram)'].append(delta)
gc.collect()
print("benchmarking lasso_path (without Gram):", end='')
sys.stdout.flush()
tstart = time()
lasso_path(X, y, precompute=False)
delta = time() - tstart
print("%0.3fs" % delta)
results['lasso_path (without Gram)'].append(delta)
return results
from sklearn.linear_model import lasso_path, enet_path
from sklearn import datasets
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
X /= X.std(axis=0) # Standardize data (easier to set the l1_ratio parameter)
# Compute paths
eps = 5e-3 # the smaller it is the longer is the path
print("Computing regularization path using the lasso...")
alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept=False)
print("Computing regularization path using the positive lasso...")
alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path(
X, y, eps, positive=True, fit_intercept=False)
print("Computing regularization path using the elastic net...")
alphas_enet, coefs_enet, _ = enet_path(
X, y, eps=eps, l1_ratio=0.8, fit_intercept=False)
print("Computing regularization path using the positve elastic net...")
alphas_positive_enet, coefs_positive_enet, _ = enet_path(
X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False)
# Display results
plt.figure(1)
