def dev():
X, Y = lassoData.lassoTrainData()
phi = design_matrix(X)
plt.plot(X, Y, 'o', label='training')
Xplot = design_matrix(np.linspace(-1, 1, 100))
wtrue = np.loadtxt('lasso_true_w.txt')
wtrue = wtrue.reshape((13, 1))
plt.plot(np.linspace(-1, 1, 100), np.dot(Xplot, wtrue), label='True model')
for lam in (.001, .01, .5, 1):
res = lasso(phi, Y, lam)
print res.coef_
plt.plot(np.linspace(-1, 1, 100),
res.predict(Xplot),
label='LASSO; $\lambda = {}$'.format(lam))
plt.xlim((-1, 1))
plt.legend(loc='best')
plt.savefig('/tmp/test.png')
min_lambda_val = 0
for lambda_val in lasso_coeffs.keys():
sse = sse_fn(lasso_coeffs[lambda_val])
all_values[lambda_val] = sse
if sse < min_sse:
min_sse = sse
min_lambda_val = lambda_val
return min_lambda_val, all_values
if __name__ == '__main__':
x_validate, y_validate = lassoData.lassoValData()
x_training, y_training = lassoData.lassoTrainData()
x_testing, y_testing = lassoData.lassoTestData()
w_true = lassoData.lassoW()
lambdas = [0, .1, .2, .3, .4, .5, .6, .7, .8, .9, 1]
M = len(w_true)
sin_basis = [
lambda x: x, lambda x: math.sin(0.4 * math.pi * x * 1),
lambda x: math.sin(0.4 * math.pi * x * 2),
lambda x: math.sin(0.4 * math.pi * x * 3),
lambda x: math.sin(0.4 * math.pi * x * 4),
lambda x: math.sin(0.4 * math.pi * x * 5),
lambda x: math.sin(0.4 * math.pi * x * 6),
lambda x: math.sin(0.4 * math.pi * x * 7),
lambda x: math.sin(0.4 * math.pi * x * 8),
lambda x: math.sin(0.4 * math.pi * x * 9),
opt_lasso_mse = mse_lasso
opt_lasso_theta = lasso_theta
if mse_ridge < opt_ridge_mse:
opt_ridge_lambda = l
opt_ridge_mse = mse_ridge
opt_ridge_theta = ridge_theta
# return [lasso,ridge]
# lasso = optimal (lambda,mse,theta)
# ridge = optimatl (lambda,mse,theta)
return [(opt_lasso_lambda, opt_lasso_mse, opt_lasso_theta),
(opt_ridge_lambda, opt_ridge_mse, opt_ridge_theta)]
if __name__ == '__main__':
x_valid, y_valid = lassoData.lassoValData()
x_train, y_train = lassoData.lassoTrainData()
x_test, y_test = lassoData.lassoTestData()
w_true = lassoData.lassoW()
M = len(w_true)
sin_basis = [
lambda x: x, lambda x: math.sin(0.4 * math.pi * x * 1),
lambda x: math.sin(0.4 * math.pi * x * 2),
lambda x: math.sin(0.4 * math.pi * x * 3),
lambda x: math.sin(0.4 * math.pi * x * 4),
lambda x: math.sin(0.4 * math.pi * x * 5),
lambda x: math.sin(0.4 * math.pi * x * 6),
lambda x: math.sin(0.4 * math.pi * x * 7),
lambda x: math.sin(0.4 * math.pi * x * 8),
lambda x: math.sin(0.4 * math.pi * x * 9),
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 28 00:40:15 2016
@author: loredp
"""
import sklearn.linear_model as skls
import lassoData
import pylab as pl
import functions as fun
import matplotlib.pyplot as plt
import numpy as np
train = lassoData.lassoTrainData()
val = lassoData.lassoValData()
test = lassoData.lassoTestData()
true_w = pl.loadtxt('lasso_true_w.txt') #True value for the data
#Step 1: transform data
X = train[0]
Y = train[1]
(Xc, Yc) = fun.center_data(X, Y)
alpha = 0.2
fig = plt.figure()
fig.add_subplot(121)
w1 = fun.compare(X, Y, M=12, alpha=alpha, basis=fun.basis_sin)
return f
#pass
return lasso_der
def der_finite_diff(step, f, x):
r = np.zeros(x.shape)
for i in range(x.size):
d = np.zeros(x.shape)
d[i] = float(step)
r[i] = (f(x + d) - f(x)) / float(step)
return r
x, y = ld.lassoTrainData()
m = 13
l = 0.1
X = compute_sinX(x, m)
loss = lasso_func_gen(X, y, l)
loss_der = lasso_der_gen(X, y, l)
B = pl.loadtxt("lasso_true_w.txt")
f = np.zeros(y.size)
for i in range(y.size):
f[i] = y[i, 0]
y = f
if __name__ == '__main__':
b, step_count = batch_gradient_descent(l * np.ones((m, 1)), 0.0001, 1e-10,
loss, loss_der, True)
def train():
lambdas = np.arange(.05, 1.15, .2)
lambdas = [0, .001, .01, .02, .03, .04] + list(lambdas)
X, Y = lassoData.lassoTrainData()
phi = design_matrix(X)
plt.figure(1)
plt.plot(X, Y, 'o', label='Training')
xx = np.linspace(-1, 1, 100)
Xplot = design_matrix(xx)
wtrue = np.loadtxt('lasso_true_w.txt')
wtrue = wtrue.reshape((13, 1))
plt.plot(xx, np.dot(Xplot, wtrue), label='True model')
results = []
ii = 0
for lam in lambdas:
res = lasso(phi, Y, lam)
results.append(res)
print res.coef_
if ii % 2 == 0:
plt.plot(xx,
res.predict(Xplot),
label='LASSO; $\lambda = {}$'.format(lam))
ii += 1
Xval, Yval = lassoData.lassoValData()
plt.plot(Xval, Yval, 'o', label='Validation')
phival = design_matrix(Xval)
#err_train = []
err_val = []
reg_penalty = []
aic = []
for res in results:
#e1 = np.sum((Y - np.dot(phi,res.coef_.reshape((13,1))).reshape((len(Y),1)))**2)
e2 = np.sum((Yval - np.dot(phival, res.coef_.reshape((13, 1))).reshape(
(len(Yval), 1)))**2)
#err_train.append(e1)
err_val.append(e2)
reg_penalty.append(np.sum(np.abs(res.coef_)))
aic.append(compute_aic(res.coef_.reshape(13), e2, len(Yval)))
#print err_train
print 'lambdas'
print lambdas
print 'SSE'
print err_val
print 'regularization'
print reg_penalty
print 'AIC'
print aic
plt.xlim((-1, 1))
plt.legend(loc='best')
plt.savefig('train_lasso.png')
plt.figure(2)
#plt.plot(lambdas, err_train, label='training error')
plt.plot(np.log(lambdas), err_val, label='SSE')
plt.plot(np.log(lambdas), reg_penalty, label='regularization penalty')
plt.plot(np.log(lambdas), aic, label='AIC')
plt.xlabel('$ln(\lambda)$')
plt.legend(loc='best')
plt.title('LASSO regularization')
plt.tight_layout()
plt.savefig('training_errors.png')
def test():
Xtrain, Ytrain = lassoData.lassoTrainData()
phi_train = design_matrix(Xtrain)
print '{} data points for training'.format(len(Xtrain))
Xval, Yval = lassoData.lassoValData()
phi_val = design_matrix(Xval)
print '{} data points for validation'.format(len(Xval))
Xtest, Ytest = lassoData.lassoTestData()
phi_test = design_matrix(Xtest)
print '{} data points for testing'.format(len(Xtest))
xx = np.linspace(-1, 1, 100)
phi_plot = design_matrix(xx)
plt.figure(3)
plt.plot(Xtrain, Ytrain, 'o', label='Training')
plt.plot(Xval, Yval, 'o', label='Validation')
plt.plot(Xtest, Ytest, 'o', label='Testing')
wtrue = np.loadtxt('lasso_true_w.txt').reshape((13, 1))
plt.plot(xx, np.dot(phi_plot, wtrue), label='True model')
lam = 0.01
res_train = lasso(phi_train, Ytrain, lam)
#res_val = lasso(phi_val,Yval,lam)
#res_test = lasso(phi_test,Ytest,lam)
plt.plot(xx, res_train.predict(phi_plot), label='LASSO')
#plt.plot(xx,res_val.predict(phi_plot),label='LASSO validation')
#plt.plot(xx,res_test.predict(phi_plot),label='LASSO testing')
lam = 0.01
res_ridge = ridge(phi_train, Ytrain, lam)
plt.plot(xx, res_ridge.predict(phi_plot), label='Ridge')
lam = 0
res_noreg = lasso(phi_train, Ytrain, lam)
plt.plot(xx, res_noreg.predict(phi_plot), label='$\lambda = 0$')
plt.legend(loc='best')
plt.xlabel('x')
plt.ylabel('y')
plt.savefig('all_fitting.png')
print 'Training error with wtrue : {:.3f}'.format(
error(wtrue, phi_train, Ytrain))
print 'Training error with w lasso: {:.3f}'.format(
error(res_train.coef_, phi_train, Ytrain))
print 'Training error with w ridge: {:.3f}'.format(
error(res_ridge.coef_, phi_train, Ytrain))
print 'Training error with w noreg: {:.3f}'.format(
error(res_noreg.coef_, phi_train, Ytrain))
print 'Validation error with wtrue : {:.3f}'.format(
error(wtrue, phi_val, Yval))
print 'Validation error with w lasso: {:.3f}'.format(
error(res_train.coef_, phi_val, Yval))
print 'Validation error with w ridge: {:.3f}'.format(
error(res_ridge.coef_, phi_val, Yval))
print 'Validation error with w noreg: {:.3f}'.format(
error(res_noreg.coef_, phi_val, Yval))
print 'Testing error with wtrue : {:.3f}'.format(
error(wtrue, phi_test, Ytest))
print 'Testing error with w lasso: {:.3f}'.format(
error(res_train.coef_, phi_test, Ytest))
print 'Testing error with w ridge: {:.3f}'.format(
error(res_ridge.coef_, phi_test, Ytest))
print 'Testing error with w noreg: {:.3f}'.format(
error(res_noreg.coef_, phi_test, Ytest))
plot_w(wtrue, 'w true', 'w_true.png')
plot_w(res_train.coef_, 'LASSO w', 'w_lasso.png')
plot_w(res_ridge.coef_, 'ridge w', 'w_ridge.png')
plot_w(res_noreg.coef_, 'unregularized w', 'w_noreg.png')