def fit(self, X: np.array, y: np.array):
"""
1 / (2 * n_samples) * ||y - Xw||^2_2
+ l1_ratio * ||w||_1
+ 0.5 * l2_ratio * ||w||^2_2
"""
if self.scale:
X, self.X_offset, self.X_scale = scale(X)
y, self.y_offset, self.y_scale = scale(y)
n_samples, n_features = X.shape
if self.fit_intercept:
ones = np.ones((n_samples, 1))
X = np.concatenate((ones, X), axis=1)
n_samples, n_features = X.shape
W = np.random.rand((n_features))
self.history = []
for _ in range(self.n_iters):
preds = X.dot(W)
dMSE = (1 / n_samples) * X.T.dot(preds - y)
dl1 = np.array(np.sign(W))
dl2 = 2 * W
W = W - self.lr * (dMSE + self.l1_ratio * dl1 +
0.5 * self.l2_ratio * dl2)
self.history.append(mse(X.dot(W), y))
self.W = W
def dummyTest():
X = np.array([[1, 1], [1, 2], [2, 2], [2, 3]])
y = np.dot(X, np.array([1, 2])) + 3
en = LinearSVR()
en.fit(X, y)
preds = en.predict(np.array([[3, 5]]))
print("Dummy MSE: " + str(mse(preds, 18)))
def testBase2(algorithm: AlgorithmBase):
from sklearn.model_selection import train_test_split
X, y = dataset(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
algorithm.fit(X_train, y_train)
preds = algorithm.predict(X_test)
res = mse(y_test, preds)
print("MSE: " + str(res))
def housingTest():
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
X, y = fetch_california_housing(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
en = ElasticNet(lr=0.1, l1_ratio=0.001, l2_ratio=0.01, n_iters=3000)
en.fit(X_train, y_train)
preds = en.predict(X_test)
res = mse(y_test, preds)
# Result is approximately 1.8
print(res)
def fit(self, X: np.array, y: np.array):
if self.scale:
X, self.X_offset, self.X_scale = scale(X)
y, self.y_offset, self.y_scale = scale(y)
n_samples, n_features = X.shape
if self.fit_intercept:
ones = np.ones((n_samples, 1))
X = np.concatenate((ones, X), axis=1)
n_samples, n_features = X.shape
if not self.gradient_descent:
self.invert(X, y)
else:
W = np.random.rand((n_features))
self.history = []
for _ in range(self.n_iters):
preds = X.dot(W)
dMSE = (1 / n_samples) * X.T.dot(preds - y)
W = W - self.lr * dMSE
self.history.append(mse(X.dot(W), y))
self.W = W
def score(self, X, y):
preds, _ = self.predict(X, True)
return mse(preds, y)
def score(self, X, y):
preds = self.predict(X)
return mse(y, preds)
self.W = W
def predict(self, X: np.array):
EPS = 1e-10
if self.scale:
X = np.array(X, dtype=np.float32)
X = (X - self.X_offset) / (self.X_scale + EPS)
if self.fit_intercept:
n_samples, n_features = X.shape
ones = np.ones((n_samples, 1))
X = np.concatenate((ones, X), axis=1)
preds = np.dot(X, self.W)
if self.scale:
preds = preds * self.y_scale + self.y_offset
return preds
def score(self, X, y):
preds = self.predict(X)
return mse(y, preds)
if __name__ == "__main__":
X = np.array([[1, 1], [1, 2], [2, 2], [2, 3]], dtype=np.float32)
y = np.dot(X, np.array([1, 2])) + 3
lr = LinearRegression(gradient_descent=False)
lr.fit(X, y)
preds = lr.predict(np.array([[3, 5]], dtype=np.float32))
print(mse(preds, [16]))
testHousing(LinearRegression(lr=1, n_iters=3000))
def score(self, X, y=None):
preds = self.inverse_transform(self.transform(X))
return mse(X, preds)
def dummyTest(algo: AlgorithmBase):
X = np.array([[1, 1], [1, 2], [2, 2], [2, 3]])
y = np.dot(X, np.array([1, 2])) + 3
algo.fit(X, y)
preds = algo.predict(np.array([[3, 5]]))
print("Dummy MSE: " + str(mse(preds, 18)))