# File Name: linregPolyVsDegree.py # Description: # Linear Regression with Polynomial Basis of different degrees # based on code code by Romain Thibaux # (Lecture 2 from http://www.cs.berkeley.edu/~asimma/294-fall06/) import numpy as np import matplotlib.pyplot as pl from utils.util import preprocessor_create from utils.util import poly_data_make from .linearRegression import linreg_fit from .linearRegression import linreg_fit_bayes from .linearRegression import linreg_predict N = 21 xtrain, ytrain, xtest, _, ytest, _ = poly_data_make(sampling='thibaux', n=N) degs = np.arange(1, 22) Nm = len(degs) # Plot error vs degree mseTrain = np.zeros(Nm) mseTest = np.zeros(Nm) for m in range(len(degs)): deg = degs[m] pp = preprocessor_create(rescale_X=True, poly=deg, add_ones=True) model = linreg_fit(xtrain, ytrain, preproc=pp) ypredTrain = linreg_predict(model, xtrain) ypredTest = linreg_predict(model, xtest) mseTrain[m] = np.mean(np.square(ytrain - ypredTrain)) mseTest[m] = np.mean(np.square(ytest - ypredTest))
import sklearn import utils.util as util from matplotlib import cm from mpl_toolkits.mplot3d import Axes3D from sklearn.linear_model import LinearRegression def gaussian(model, x, y, sigma): xx, yy = np.meshgrid(x, y) yypred = model.coef_ * xx + model.intercept_ z = (yy - yypred) * (yy - yypred) z = np.exp(-0.5 * z) return z / ((2 * np.pi * sigma) ** 0.5) N = 21 xtrain, ytrain, xtest, _, _, _ = util.poly_data_make(sampling='thibaux', n=N) lin = LinearRegression(fit_intercept=True) model = lin.fit(xtrain.reshape((N, 1)), ytrain) ypred = model.predict(xtest.reshape((len(xtest), 1))) fig = pl.figure() ax = Axes3D(fig) xrange = np.linspace(min(xtest), max(xtest), 300) yrange = np.linspace(min(ypred), max(ypred), 300) sigma = 1 z = gaussian(lin, xrange, yrange, sigma) xx, yy = np.meshgrid(xrange, yrange) surf = ax.plot_surface(
#Function to expand from x to design matrix of degree deg def ExpandtoDeg(x, deg): return np.array([x**i for i in range(deg + 1) ]).transpose().reshape(-1, deg + 1) for ModDeg in degrees: ns = np.round(np.linspace(10, 210, 20)) err = [] errtrain = [] for n in ns: #Forming data xtrain, ytrain, xtest, _, ytest, _ = poly_data_make(sampling='thibaux', deg=TrueDeg, n=n) #Rescaling data scaler = MinMaxScaler(feature_range=(-1, 1)) xtrain = scaler.fit_transform(xtrain.reshape(-1, 1)) xtest = scaler.transform(xtest.reshape(-1, 1)) #Fitting ridge regression. Small differences in alpha near zero make a visual difference in the plot when n is close to 0. regr = Ridge( alpha=0, fit_intercept=False ) #Using ridge instead of ordinary least squares for numerical stability XDesignTrain = ExpandtoDeg(xtrain, ModDeg) XDesignTest = ExpandtoDeg(xtest, ModDeg) regr.fit(XDesignTrain, ytrain) ypred = regr.predict(XDesignTest)
def contoursSSEDemo(): N = 21 x,y,_,_,_,_ = util.poly_data_make(sampling='thibaux', n=N) X = util.add_ones(x) return X,y
#!/usr/bin/env python3 # Linear regression on data with residuals. import matplotlib.pyplot as pl import numpy as np import utils.util as util N = 21 x, y, _, _, _, _ = util.poly_data_make(sampling='thibaux', n=N) X = np.concatenate((np.ones((N,1)), x.reshape(N,1)), axis=1) w = np.linalg.lstsq(X, y)[0] y_estim = np.dot(X,w) pl.plot(X[:,1], y, 'o') pl.plot(X[:,1], y_estim, '-') pl.savefig('linregResidualsNoBars.png') pl.show() for x0, y0, y_hat in zip(X[:,1], y, y_estim): pl.plot([x0, x0],[y0, y_hat],'k-') pl.plot(X[:,1], y, 'o') pl.plot(X[:,1], y_estim, '-') pl.savefig('linregResidualsBars.png') pl.show()
#!/usr/bin/env python3 # Linear regression on data with residuals. import matplotlib.pyplot as pl import numpy as np from utils import util N = 21 x, y, _, _, _, _ = util.poly_data_make(sampling='thibaux', n=N) X = np.concatenate((np.ones((N, 1)), x.reshape(N, 1)), axis=1) w = np.linalg.lstsq(X, y)[0] y_estim = np.dot(X, w) pl.plot(X[:, 1], y, 'o') pl.plot(X[:, 1], y_estim, '-') pl.savefig('linregResidualsNoBars.png') pl.show() for x0, y0, y_hat in zip(X[:, 1], y, y_estim): pl.plot([x0, x0], [y0, y_hat], 'k-') pl.plot(X[:, 1], y, 'o') pl.plot(X[:, 1], y_estim, '-') pl.savefig('linregResidualsBars.png') pl.show()