def linreg_fit(X, y, **kwargs):
"""
Fit a linear regression model with MLE or MAP.
This is a port of linregFit.m from pmtk3.
:param X: N*D design matrix
:param y: N*1 response vector
"""
pp = util.preprocessor_create(add_ones=True, standardize_X=False) # default
N = len(X)
D = 1 if len(X.shape) < 2 else X.shape[1]
weights = kwargs['weights'] if 'weights' in kwargs else np.ones(N)
reg_type = kwargs['reg_type'] if 'reg_type' in kwargs else None
likelihood = kwargs['likelihood'] if 'likelihood' in kwargs else 'gaussian'
lambda_ = kwargs['lambda_'] if 'lambda_' in kwargs else None
fit_options = kwargs['fit_options'] if 'fit_options' in kwargs else None
preproc = kwargs['preproc'] if 'preproc' in kwargs else pp
fit_fn_name = kwargs['fit_fn_name'] if 'fit_fn_name' in kwargs else None
winit = kwargs['winit'] if 'winit' in kwargs else None
if preproc is None:
preproc = util.preprocessor_create()
if preproc.add_ones:
D += 1
if winit is None:
winit = np.zeros(D)
if reg_type is None:
if lambda_ is None:
# MLE
reg_type = 'l2'
lambda_ = 0
else:
#L2
reg_type = 'l2'
if fit_options is None:
fit_options = default_fit_options(reg_type, D)
if fit_fn_name is None:
if reg_type.lower() == 'l1':
fit_fn_name = 'l1GeneralProjection'
elif reg_type.lower() == 'l2':
fit_fn_name = 'qr'
model = {}
if likelihood.lower() == 'huber':
raise NotImplementedError
elif likelihood.lower() == 'student':
raise NotImplementedError
elif likelihood.lower() == 'gaussian':
preproc, X = preprocessor_apply_to_train(preproc, X)
N = len(X)
D = 1 if len(X.shape) < 2 else X.shape[1]
model['lambda_'] = lambda_
lambda_vec = lambda_ * np.ones(D)
if preproc.add_ones:
lambda_vec[0] = 0 # don't penalize bias term
winit = np.zeros(D)
opts = fit_options
if reg_type == 'l1':
raise NotImplementedError
elif reg_type == 'l2':
if fit_fn_name == 'qr':
if lambda_ == 0:
R = np.diag(np.sqrt(weights))
RX = R.dot(X)
w = np.linalg.pinv(RX.T.dot(RX)).dot(RX.T).dot(R.dot(y))
else:
raise NotImplementedError
elif fit_fn_name == 'minfunc':
raise NotImplementedError
else:
raise ValueError('Invalid fit function')
elif reg_type == 'scad':
raise NotImplementedError
else:
raise ValueError('Invalid regression type')
else:
raise ValueError('Invalid likelihood')
model['w'] = w
yhat = X.dot(w)
if weights.sum() == 0:
model['sigma2'] = np.spacing(1)
else:
model['sigma2'] = np.sum(weights * np.square(y - yhat)) / \
np.sum(weights)
model['preproc'] = preproc
model['model_type'] = 'linreg'
model['likelihood'] = likelihood
return model
from .linearRegression import linreg_fit
from .linearRegression import linreg_fit_bayes
from .linearRegression import linreg_predict
N = 21
xtrain, ytrain, xtest, _, ytest, _ = poly_data_make(sampling='thibaux', n=N)
degs = np.arange(1, 22)
Nm = len(degs)
# Plot error vs degree
mseTrain = np.zeros(Nm)
mseTest = np.zeros(Nm)
for m in range(len(degs)):
deg = degs[m]
pp = preprocessor_create(rescale_X=True, poly=deg, add_ones=True)
model = linreg_fit(xtrain, ytrain, preproc=pp)
ypredTrain = linreg_predict(model, xtrain)
ypredTest = linreg_predict(model, xtest)
mseTrain[m] = np.mean(np.square(ytrain - ypredTrain))
mseTest[m] = np.mean(np.square(ytest - ypredTest))
ndx = degs <= 16
fig = pl.figure()
pl.plot(degs[ndx], mseTrain[ndx], lw=3)
pl.plot(degs[ndx], mseTest[ndx], lw=3)
pl.xlabel('degree')
pl.ylabel('mse')
leg = pl.legend(('train', 'test'), loc='upper left')
leg.draw_frame(False)
pl.savefig('linregPolyVsDegreeUcurve.png')
from SupervisedModels.linearRegression import linreg_fit
from SupervisedModels.linearRegression import linreg_fit_bayes
from SupervisedModels.linearRegression import linreg_predict
N = 21
xtrain, ytrain, xtest, _, ytest, _ = poly_data_make(sampling='thibaux', n=N)
degs = np.arange(1, 22)
Nm = len(degs)
# Plot error vs degree
mseTrain = np.zeros(Nm)
mseTest = np.zeros(Nm)
for m in xrange(len(degs)):
deg = degs[m]
pp = preprocessor_create(rescale_X=True, poly=deg, add_ones=True)
model = linreg_fit(xtrain, ytrain, preproc=pp)
ypredTrain = linreg_predict(model, xtrain)
ypredTest = linreg_predict(model, xtest)
mseTrain[m] = np.mean(np.square(ytrain - ypredTrain))
mseTest[m] = np.mean(np.square(ytest - ypredTest))
ndx = degs <= 16
fig = pl.figure()
pl.plot(degs[ndx], mseTrain[ndx], lw=3)
pl.plot(degs[ndx], mseTest[ndx], lw=3)
pl.xlabel('degree')
pl.ylabel('mse')
leg = pl.legend(('train', 'test'), loc='upper left')
leg.draw_frame(False)
pl.savefig('linregPolyVsDegreeUcurve.png')
