-
Notifications
You must be signed in to change notification settings - Fork 0
/
linear_regression.py
246 lines (198 loc) · 8.07 KB
/
linear_regression.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
import numpy as np
import sklearn
from sklearn.base import BaseEstimator, RegressorMixin, TransformerMixin
from sklearn.preprocessing import PolynomialFeatures
from pandas import DataFrame
from sklearn.utils import check_array
from sklearn.utils.validation import check_is_fitted, check_X_y
class LinearRegressor(BaseEstimator, RegressorMixin):
"""
Implements Linear Regression prediction and closed-form parameter fitting.
"""
def __init__(self, reg_lambda=0.1):
self.reg_lambda = reg_lambda
def predict(self, X):
"""
Predict the class of a batch of samples based on the current weights.
:param X: A tensor of shape (N,n_features_) where N is the batch size.
:return:
y_pred: np.ndarray of shape (N,) where each entry is the predicted
value of the corresponding sample.
"""
X = check_array(X)
check_is_fitted(self, 'weights_')
# TODO: Calculate the model prediction, y_pred
y_pred = None
# ====== YOUR CODE: ======
y_pred = X.dot(self.weights_)
# ========================
return y_pred
def fit(self, X, y):
"""
Fit optimal weights to data using closed form solution.
:param X: A tensor of shape (N,n_features_) where N is the batch size.
:param y: A tensor of shape (N,) where N is the batch size.
"""
X, y = check_X_y(X, y)
# TODO:
# Calculate the optimal weights using the closed-form solution
# Use only numpy functions. Don't forget regularization.
w_opt = None
# ====== YOUR CODE: ======
xt_x = np.transpose(X).dot(X)
lambda_mat = self.reg_lambda*np.identity(X.shape[1])
lambda_mat[0, 0] = 0
psuedo_mat = np.linalg.inv(xt_x + lambda_mat * X.shape[0])
w_opt = psuedo_mat.dot(np.transpose(X).dot(y))
# ========================
self.weights_ = w_opt
return self
def fit_predict(self, X, y):
return self.fit(X, y).predict(X)
class BiasTrickTransformer(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X: np.ndarray):
"""
:param X: A tensor of shape (N,D) where N is the batch size and D is
the number of features.
:returns: A tensor xb of shape (N,D+1) where xb[:, 0] == 1
"""
X = check_array(X, ensure_2d=True)
# TODO:
# Add bias term to X as the first feature.
# See np.hstack().
xb = None
# ====== YOUR CODE: ======
ones = np.ones(shape=(X.shape[0], 1))
xb = np.hstack((ones, X))
# ========================
return xb
class BostonFeaturesTransformer(BaseEstimator, TransformerMixin):
"""
Generates custom features for the Boston dataset.
"""
def __init__(self, degree=2):
self.degree = degree
# TODO: Your custom initialization, if needed
# Add any hyperparameters you need and save them as above
# ====== YOUR CODE: ======
# raise NotImplementedError()
# ========================
def fit(self, X, y=None):
return self
def transform(self, X):
"""
Transform features to new features matrix.
:param X: Matrix of shape (n_samples, n_features_).
:returns: Matrix of shape (n_samples, n_output_features_).
"""
X = check_array(X)
# TODO:
# Transform the features of X into new features in X_transformed
# Note: You CAN count on the order of features in the Boston dataset
# (this class is "Boston-specific"). For example X[:,1] is the second
# feature ('ZN').
X_transformed = None
# ====== YOUR CODE: ======
poly = PolynomialFeatures(self.degree)
X_transformed = poly.fit_transform(X[:, 1:])
# ========================
return X_transformed
def top_correlated_features(df: DataFrame, target_feature, n=5):
"""
Returns the names of features most strongly correlated (correlation is
close to 1 or -1) with a target feature. Correlation is Pearson's-r sense.
:param df: A pandas dataframe.
:param target_feature: The name of the target feature.
:param n: Number of top features to return.
:return: A tuple of
- top_n_features: Sequence of the top feature names
- top_n_corr: Sequence of correlation coefficients of above features
Both the returned sequences should be sorted so that the best (most
correlated) feature is first.
"""
# TODO: Calculate correlations with target and sort features by it
# ====== YOUR CODE: ======
table = df.corrwith(other=df[target_feature], method='pearson')
table_abs = table.abs()
top_n_features = table_abs.nlargest(n=n+1).sort_values(ascending=False)
top_n_features = [feature[0] for feature in top_n_features[1:].items()]
top_n_corr = [table.get(f) for f in top_n_features]
# ========================
return top_n_features, top_n_corr
def mse_score(y: np.ndarray, y_pred: np.ndarray):
"""
Computes Mean Squared Error.
:param y: Predictions, shape (N,)
:param y_pred: Ground truth labels, shape (N,)
:return: MSE score.
"""
# TODO: Implement MSE using numpy.
# ====== YOUR CODE: ======
y_residual = y - y_pred
y_residual = np.power(y_residual, 2)
mse = (1/y.shape[0]) * np.sum(y_residual)
# ========================
return mse
def r2_score(y: np.ndarray, y_pred: np.ndarray):
"""
Computes R^2 score,
:param y: Predictions, shape (N,)
:param y_pred: Ground truth labels, shape (N,)
:return: R^2 score.
"""
# TODO: Implement R^2 using numpy.
# ====== YOUR CODE: ======
y_residual = y - y_pred
y_residual = np.power(y_residual, 2)
y_mean = np.average(y)
y_var = y - y_mean
y_var = np.power(y_var, 2)
r2 = 1 - (np.sum(y_residual) / np.sum(y_var))
# ========================
return r2
def cv_best_hyperparams(model: BaseEstimator, X, y, k_folds,
degree_range, lambda_range):
"""
Cross-validate to find best hyperparameters with k-fold CV.
:param X: Training data.
:param y: Training targets.
:param model: sklearn model.
:param lambda_range: Range of values for the regularization hyperparam.
:param degree_range: Range of values for the degree hyperparam.
:param k_folds: Number of folds for splitting the training data into.
:return: A dict containing the best model parameters,
with some of the keys as returned by model.get_params()
"""
# TODO: Do K-fold cross validation to find the best hyperparameters
# Notes:
# - You can implement it yourself or use the built in sklearn utilities
# (recommended). See the docs for the sklearn.model_selection package
# http://scikit-learn.org/stable/modules/classes.html#module-sklearn.model_selection
# - If your model has more hyperparameters (not just lambda and degree)
# you should add them to the search.
# - Use get_params() on your model to see what hyperparameters is has
# and their names. The parameters dict you return should use the same
# names as keys.
# - You can use MSE or R^2 as a score.
# ====== YOUR CODE: ======
# params = model.get_params()
kf = sklearn.model_selection.KFold(n_splits=k_folds)
params_grid = {'bostonfeaturestransformer__degree': degree_range, 'linearregressor__reg_lambda': lambda_range}
min_acc = np.inf
for params in list(sklearn.model_selection.ParameterGrid(params_grid)):
model.set_params(**params)
curr_acc = 0
for train_idx, test_idx in kf.split(X):
train_x, train_y = X[train_idx], y[train_idx]
test_x, test_y = X[test_idx], y[test_idx]
model.fit(train_x, train_y)
y_pred = model.predict(test_x)
curr_acc += mse_score(test_y, y_pred)
mean = curr_acc/k_folds
if mean < min_acc:
min_acc=mean
best_params = params
# ========================
return best_params