def fit(self, X, y, X_val=None, y_val=None): X = normalize(polynomial_features(X, degree=self.degree)) X_val = None if X_val is None else normalize( polynomial_features(X_val, degree=self.degree)) super(PolynomialRegression, self).fit(X=X, y=y, X_val=X_val, y_val=y_val)
def test_polynomial_features(): from utils import polynomial_features x = np.random.random((3, 2)).tolist() result = [] for k in [2, 3]: result.append('[TEST polynomial_features],' + weights_to_string(polynomial_features(x, k=k))) return result
def test_data_procession(self): features, values = generate_data_part_3() train_features, train_values = features[:100], values[:100] valid_features, valid_values = features[100:120], values[100:120] test_features, test_values = features[120:], values[120:] assert len(train_features) == len(train_values) == 100 assert len(valid_features) == len(valid_values) == 20 assert len(test_features) == len(test_values) == 30 best_mse, best_k = 1e10, -1 for k in [1, 3, 10]: train_features_extended = polynomial_features(train_features, k) model = LinearRegression(nb_features=k) model.train(train_features_extended, train_values) train_mse = mean_squared_error(train_values, model.predict(train_features_extended)) valid_features_extended = polynomial_features(valid_features, k) valid_mse = mean_squared_error(valid_values, model.predict(valid_features_extended)) print(f'[part 1.4.1]\tk: {k:d}\t' f'train mse: {train_mse:.5f}\tvalid mse: {valid_mse:.5f}') if valid_mse < best_mse: best_mse, best_k = valid_mse, k
def test_add_polynomial_feature(self): features, values = generate_data_part_2() plt.scatter([x[0] for x in features], values, label='origin'); for k in [2, 4, 10]: # TODO: confirm polynomial feature and k = Xk features_extended = polynomial_features(features, k) model = LinearRegression(nb_features=k) model.train(features_extended, values) mse = mean_squared_error(values, model.predict(features_extended)) print(f'[part 1.3.2]\tk: {k:d}\tmse: {mse:.5f}') plt.plot([x[0] for x in features], model.predict(features_extended), label=f'k={k}'); plt.legend() plt.show()
def predict(self, X_test, eti=False): X_test = np.array(X_test) #if polynominal features if self.poly_degree: X_test = polynomial_features(X_test, degree=self.poly_degree) y_pred = np.dot(X_test, self.w) #if the lower and upper boundaries for the 95% equal tail interval should be returned if eti: lower_w = self.eti[:, 0] upper_w = self.eti[:, 1] y_lower_pred = np.dot(X_test, lower_w) y_upper_pred = np.dot(X_test, upper_w) return y_pred, y_lower_pred, y_upper_pred return y_pred
def fit(self, X, y): #if polynomial transformation if self.poly_degree: X = polynomial_features(X, degree=self.poly_degree) n_samples, n_features = np.shape(X) X_X = X.T.dot(X) #least squares approximate of beta beta_hat = np.linalg.pinv(X_X).dot(X.T).dot(y) # the posterior parameters can be determined analytically # since we assume conjugate priors for the likelihoods. #normal prior / likelihood => Normal posterior mu_n = np.linalg.pinv(X_X + self.omega0).dot( X_X.dot(beta_hat) + self.omega0.dot(self.mu0)) omega_n = X_X + self.omega0 #scaled inverse chi-squared prior / likelihood => scaled inverse chi-squared posterior nu_n = self.nu0 + n_samples sigma_sq_n = (1.0 / nu_n) * ( self.nu0 * self.sigma_sq0 + (y.T.dot(y) + self.mu0.T.dot(self.omega0).dot(self.mu0) - mu_n.T.dot(omega_n.dot(mu_n)))) # simulate parameter values for n_draws beta_draws = np.empty((self.n_draws, n_features)) for i in range(self.n_draws): sigma_sq = self._draw_scaled_inv_chi_sq(n=1, df=nu_n, scale=sigma_sq_n) beta = multivariate_normal.rvs(size=1, mean=mu_n[:, 0], cov=sigma_sq * np.linalg.pinv(omega_n)) #save parameter draws beta_draws[i, :] = beta #Select the mean of the simulated variables as the ones used to make predictions self.w = np.mean(beta_draws, axis=0) # lower and upper boundary of the credible interval l_eti = 50 - self.cred_int / 2 u_eti = 50 - self.cred_int / 2 self.eti = np.array([[ np.percentile(beta_draws[:, i], q=l_eti), np.percentile(beta_draws[:, i], q=u_eti) ] for i in range(n_features)])
def predict(self, X, eti=False): #if polynomial transformation if self.poly_degree: X = polynomial_features(X, degree=self.poly_degree) y_pred = X.dot(self.w) # if the lower and upper doundaries for the 95% # equal tail interval should be returned if eti: lower_w = self.eti[:, 0] upper_w = self.eti[:, 1] y_lower_pred = X.dot(lower_w) y_upper_pred = X.dot(upper_w) return y_pred, y_lower_pred, y_upper_pred return y_pred
def fit(self, X, y): X = normalize(polynomial_features(X, degree=self.degree)) # Insert constant ones for bias weights X = np.insert(X, 0, 1, axis=1) self.training_errors = [] n_features = X.shape[1] # Initialize weights randomly [-1/N, 1/N] limit = 1 / math.sqrt(n_features) self.w = np.random.uniform(-limit, limit, (n_features, )) # Do gradient descent for n_iterations for i in range(self.n_iterations): y_pred = X.dot(self.w) # Calculate l2 loss mse = np.mean(0.5 * (y - y_pred)**2 + self.alpha * np.linalg.norm(self.w)) self.training_errors.append(mse) # Gradient of l2 loss w.r.t w grad_w = -(y - y_pred).dot(X) + self.alpha * np.sign(self.w) # Update the weights self.w -= self.learning_rate * grad_w
print('[part 1.3.1]\tmse: {mse:.5f}'.format(mse=mse)) plt.scatter([x[0] for x in features], values, label='origin'); plt.plot([x[0] for x in features], model.predict(features), label='predicted'); plt.legend() from utils import polynomial_features features, values = generate_data_part_2() features = data[2] values = data[3] plt.scatter([x[0] for x in features], values, label='origin'); for k in [2, 4, 8]: features_extended = polynomial_features(features, k) #print(features_extended) model = LinearRegression(nb_features=k) model.train(features_extended, values) mse = mean_squared_error(values, model.predict(features_extended)) print('[part 1.3.2]\tk: {k:d}\tmse: {mse:.5f}'.format(k=k, mse=mse)) plt.plot([x[0] for x in features], model.predict(features_extended), label='k={k}'.format(k=k)); plt.legend() from data import generate_data_part_3 features, values = generate_data_part_3() features = data[4] values = data[5] train_features, train_values = features[:100], values[:100]
def predict(self, X): X = normalize(polynomial_features(X, degree=self.degree)) return super(ElasticNet, self).predict(X)
def fit(self, X, y): X = normalize(polynomial_features(X, degree=self.degree)) super(ElasticNet, self).fit(X, y)
def predict(self, X): X = normalize(polynomial_features(X, degree=self.degree)) return super(PolynomialRidgeRegression, self).predict(X)
def fit(self, X, y): X = normalize(polynomial_features(X, degree=self.degree)) super(PolynomialRidgeRegression, self).fit(X, y)
def predict(self, X): X = polynomial_features(X, degree=self.degree) return super(PolynomialRegression, self).predict(X)
def fit(self, X, y): X = polynomial_features(X, degree=self.degree) super(PolynomialRegression, self).fit(X, y)
def test_polynomial_features(self): features = [[1.0, 2, 3], [4.0, 5, 6], [7.0, 8, 9]] assert_array_almost_equal( numpy.array([[1.0, 2, 3, 1, 4, 9], [4.0, 5, 6, 16, 25, 36], [7.0, 8, 9, 49, 64, 81]]), numpy.array(polynomial_features(features, 2)))
def predict(self, X_test): X_test = normalize(polynomial_features(X_test, degree=self.degree)) super(PolynomialRegression, self).predict(X_test=X_test)
def fit(self, X, y): X = polynomial_features(X, degree=self.degree) super().fit(X, y)
def predict(self, X): X = normalize(polynomial_features(X, degree=self.degree)) # Insert constant ones for bias weights X = np.insert(X, 0, 1, axis=1) y_pred = X.dot(self.w) return y_pred