def get_ridge_regression_weights(X, Y, alphas): ridge = Ridge(normalize=True) coefs = [] for a in alphas: ridge.set_params(alpha=a) ridge.fit(X, Y) coefs.append(ridge.coef_) ax = plt.gca() ax.plot(alphas, coefs) ax.set_xscale('log') plt.axis('tight') plt.xlabel('alpha') plt.ylabel('weights') plt.plot(alphas, coefs) plt.title("Weight of Features as Alpha Increases") plt.show() for a in alphas: X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.5, random_state=1) ridge = Ridge(alpha=a, normalize=True) ridge.fit(X_train, y_train) pred2 = ridge.predict(X_test) print("alpha: ", a) print("Ridge Coeffecients") print(pd.Series(ridge.coef_, index=X.columns)) print("MSE: " + str(mean_squared_error(y_test, pred2))) print("*******************************")
def Ridge_model(train_linear, test_linear): ridgecv = RidgeCV(alphas = np.logspace(-5, 4, 400)) ridgecv.fit(train_linear_fea, train_linear_tar) ridgecv_score = ridgecv.score(train_linear_fea, train_linear_tar) ridgecv_alpha = ridgecv.alpha_ print("Best alpha : ", ridgecv_alpha, "Score: ",ridgecv_score) coef=pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending =False) start=time.time() ridge =Ridge(normalize = True) ridge.set_params(alpha=ridgecv_alpha,max_iter = 10000) #ridge.set_params(alpha=6,max_iter = 10000) ridge.fit(x_train, y_train) end=time.time() mean_squared_error(y_test, ridge.predict(x_test)) coef_ridge=pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending =False) evaluate(ridge,x_test,y_test,x_train,y_train) print('Time elapsed: %.4f seconds' % (end-start)) y_ridge_predict=ridge.predict(train_linear_fea) x_line = np.arange(700000) y_line=x_line plt.scatter(real_train_tar,np.expm1(y_ridge_predict)) plt.plot(x_line, y_line, color='r') plt.xlabel('Actual Sale Price') plt.ylabel('Predict Sle Price') test_prediction_ridge=np.expm1(ridge.predict(test_linear)) write_pkl(ridgecv_alpha, '/Users/vickywinter/Documents/NYC/Machine Learning Proj/Pickle/ridge_params.pkl') return test_prediction_ridge
def create_reg(self): if self.linear_model == 'linear_regression': return LinearRegression() elif self.linear_model == 'ridge': model = Ridge(self.alpha) model.set_params(**self.params) return model
def Ridge_model(train_linear, test_linear): ridgecv = RidgeCV(alphas=np.logspace(-5, 4, 400)) ridgecv.fit(train_linear_fea, train_linear_tar) ridgecv_score = ridgecv.score(train_linear_fea, train_linear_tar) ridgecv_alpha = ridgecv.alpha_ print("Best alpha : ", ridgecv_alpha, "Score: ", ridgecv_score) coef = pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending=False) start = time.time() ridge = Ridge(normalize=True) ridge.set_params(alpha=ridgecv_alpha, max_iter=10000) #ridge.set_params(alpha=6,max_iter = 10000) ridge.fit(x_train, y_train) end = time.time() mean_squared_error(y_test, ridge.predict(x_test)) coef_ridge = pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending=False) evaluate(ridge, x_test, y_test, x_train, y_train) print('Time elapsed: %.4f seconds' % (end - start)) y_ridge_predict = ridge.predict(train_linear_fea) x_line = np.arange(700000) y_line = x_line plt.scatter(real_train_tar, np.expm1(y_ridge_predict)) plt.plot(x_line, y_line, color='r') plt.xlabel('Actual Sale Price') plt.ylabel('Predict Sle Price') test_prediction_ridge = np.expm1(ridge.predict(test_linear)) write_pkl( ridgecv_alpha, '/Users/vickywinter/Documents/NYC/Machine Learning Proj/Pickle/ridge_params.pkl' ) return test_prediction_ridge
def linear_ridge2(): X = 1. / (np.arange(1, 11) + np.arange(0, 10)[:, np.newaxis]) y = np.ones(10) n_alphas = 200 # alphas count is 200, 都在10的-10次方和10的-2次方之间 alphas = np.logspace(-10, -2, n_alphas) clf = Ridge(fit_intercept=False) coefs = [] # 循环200次 for a in alphas: # 设置本次循环的超参数 clf.set_params(alpha=a) # 针对每个alpha做ridge回归 clf.fit(X, y) # 把每一个超参数alpha对应的theta存下来 coefs.append(clf.coef_) ax = plt.gca() ax.plot(alphas, coefs) # 将alpha的值取对数便于画图 ax.set_xscale('log') # 翻转x轴的大小方向,让alpha从大到小显示 ax.set_xlim(ax.get_xlim()[::-1]) plt.xlabel('alpha') plt.ylabel('weights') plt.title('Ridge coefficients as a function of the regularization') plt.axis('tight') plt.show()
def ridge_reg(): n_alphas = 200 ridge_alphas = np.logspace(-2, 6, n_alphas) ridge_cv = RidgeCV(alphas=ridge_alphas, scoring='neg_mean_squared_error', cv=3) ridge_cv.fit(X, y) print('------------') print(ridge_cv.coef_, "\n", ridge_cv.alpha_) # Выведем влияние альфы (от наибольшей к наименьшей) для каждого признака model = Ridge() coefs = [] for a in ridge_alphas: model.set_params(alpha=a) model.fit(X, y) coefs.append(model.coef_) ax = plt.gca() ax.plot(ridge_alphas, coefs) ax.set_xscale('log') ax.set_xlim(ax.get_xlim()[::-1]) # reverse axis plt.xlabel('alpha') plt.ylabel('weights') plt.title('Ridge coefficients as a function of the regularization') plt.axis('tight') plt.show()
def _ridge_search(X, Y, train, test, alphas=np.logspace(0, 4, 20), scoring=corr): """Fit ridge regression sweeping through alphas Parameters ---------- X : array (n_samples, n_predictors) design matrix/predictors Y : array (n_samples, n_features) response matrix train : array index array for training test : array index array for testing alphas : array alpha values used to fit scoring : callable function used to score the fit (default correlation) Returns ------- scores : array (n_alphas, n_features) score of prediction for each alpha and each feature """ ridge = Ridge(fit_intercept=False, solver='svd') scores = [] for alpha in alphas: ridge.set_params(alpha=alpha) ridge.fit(X[train], Y[train]) scores.append(scoring(Y[test].T, ridge.predict(X[test]).T)) return np.array(scores)
def fit_data(X, Y): print("Fitting started...") from sklearn.linear_model import Ridge, RidgeCV, Lasso, LassoCV print("Least Square") coef = np.linalg.lstsq(X, Y, rcond=None)[0] print(coef) #print(intercept) print("Ridge") #ridgecv = RidgeCV(alphas = None, cv = 10) #ridgecv.fit(X, Y) rr100 = Ridge(alpha=100) rr100.fit(X, Y) print(rr100.coef_) print(rr100.intercept_) print("LASSO") lassocv = LassoCV(alphas = None, cv = 10, max_iter = 100000) lassocv.fit(X, Y) rr100 = Lasso(max_iter=10000) # comparison with alpha value rr100.set_params(alpha=lassocv.alpha_) rr100.fit(X, Y) print(rr100.coef_) print(rr100.intercept_) print("")
def load_regressor_Ridge(X_train, y_train): best_estimator_ridge = joblib.load('./built_models/regressor_ridge.pkl') regressor_ridge = Ridge() regressor_ridge.set_params(**best_estimator_ridge) print regressor_ridge regressor_ridge.fit(X_train, y_train) return regressor_ridge
def ridge_coefs(X, y, alphas): coefs = [] ridge_reg = Ridge() for a in alphas: ridge_reg.set_params(alpha=a) ridge_reg.fit(X, y) coefs.append(ridge_reg.coef_[0]) return coefs
def non_zero_coeff(X, y, _lambda): clf = Ridge(normalize=True, solver='sag') # clf = Lasso() clf.set_params(alpha=_lambda) clf.fit(X, y) nonzero = np.count_nonzero(clf.coef_) print("Number of non-zero features: " + str(nonzero) + ", alpha: " + str(_lambda)) return nonzero, (clf.coef_ != 0)
def make_ridge_pred(df, next_week, debug=0): """ This method creates predictions using ridge regression. """ #Tuned## params_old = { 'alpha': 10, 'max_iter': 1, 'solver': 'cholesky', 'normalize': True, 'fit_intercept': True } params = { 'alpha': 10, 'max_iter': -1, 'solver': 'sparse_cg', 'normalize': False, 'fit_intercept': True } rand_space = { 'alpha': [1e-10, 1e-5, 1e-2, 1e-1, 1, 10, 100], 'normalize': [True, False], 'fit_intercept': [True, False], 'solver': ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'], 'max_iter': [int(x) for x in np.linspace(1, 1000, num=10)] } space = { 'alpha': [8, 9, 10, 11], 'normalize': [True, False], 'fit_intercept': [True, False], 'solver': ['cholesky'], 'max_iter': [None, 1, 2, 10] } X_train, X_test, Y_train, Y_test = process_data(df, next_week) multi_ridge = Ridge() multi_ridge.set_params(**params) #best_random = random_search(multi_ridge, rand_space, next_week, 100, 3, X_train, Y_train) #best_random = grid_search(multi_ridge, space, next_week, 3, X_train, Y_train) multi_ridge.fit(X_train, Y_train) next_week[Y_train.columns] = multi_ridge.predict( next_week[X_train.columns]) if debug: y_pred_untrain = multi_ridge.predict(X_train) print(next_week) print("Score: ", multi_ridge.score(X_train, Y_train) * 100) print("MSE: ", metrics.mean_squared_error(Y_train, y_pred_untrain)) print( "CV: ", ms.cross_val_score(multi_ridge, Y_train, y_pred_untrain, cv=10, scoring='neg_mean_squared_error')) return next_week
class Ridge(Model): def create_model(self): self.ridge = RidgeLR() def fit(self, train_x, train_y): self.ridge.fit(train_x, train_y) def set_config(self, config): self.ridge.set_params(**config) def predict(self, test_x): return self.ridge.predict(test_x)
def plot_regularizations(X_train, X_test, y_train, y_test, alphas): # Compute train and test errors regressor = Ridge() # regularized linear regression train_errors = list() test_errors = list() for alpha in alphas: print("Woring on alpha = %s..." % alpha) regressor.set_params(alpha=alpha) regressor.fit(X_train, y_train) train_error = regressor.score(X_train, y_train) print("train_error : %s..." % train_error) train_errors.append(train_error) test_error = regressor.score(X_test, y_test) print("test_error : %s..." % test_error) test_errors.append(test_error) i_alpha_optim = np.argmax(test_errors) alpha_optim = alphas[i_alpha_optim] print("Optimal regularization parameter : %s" % alpha_optim) # Estimate the coef_ on full data with optimal regularization parameter regressor.set_params(alpha=alpha_optim) coef_ = regressor.fit(X_train, y_train).coef_ # ############################################################################# # Plot results functions import matplotlib.pyplot as plt plt.subplot(2, 1, 1) plt.semilogx(alphas, train_errors, label='Train') plt.semilogx(alphas, test_errors, label='Test') plt.vlines(alpha_optim, plt.ylim()[0], np.max(test_errors), color='k', linewidth=3, label='Optimum on test') plt.legend(loc='lower left') plt.ylim([0, 1.2]) plt.xlabel('Regularization parameter') plt.ylabel('Performance') # Show estimated coef_ vs true coef plt.subplot(2, 1, 2) plt.plot(coef_, label='Estimated coef') plt.legend() plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.26) plt.show()
def LinearModelRidge(X_train, y_train, X_test, y_test): alphas = 10**np.linspace(10, -2, 100) * 0.5 ridgecv = RidgeCV(alphas=alphas, scoring="neg_mean_squared_error", cv=10) ridgecv.fit(X_train, y_train) print("Value of lambda ", ridgecv.alpha_) ridge = Ridge() ridge.set_params(alpha=ridgecv.alpha_) ridge.fit(X_train, y_train) print_evaluation_metrics(ridge, "Ridge Model", X_train, y_train, X_test, y_test)
def feature_selection(df, target, test_split, α=5, thresh=50): ### Bich-Tien PHAN """ :param df: (pandas DataFrame) data to process :param target: (string) label name to predict :param test_split: (float) percentage of test data :param α: (float) ridge regression hyperparameter, the maximal value to determine the optimal value of alpha for ridge regression :param thresh: (float) parameter to select the significant features :return: list of colums to keep after ridge regression """ #Prepocessing for ridge regression X = df.drop(columns=target) X = preprocessing.scale(X) y = df[target].to_numpy() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_split) coefs = [] error = [] ridge_reg = Ridge() n_alphas = 200 alphas = np.logspace(-5, α, n_alphas) # iterate lambdas for alpha in alphas: # training ridge_reg.set_params(alpha=alpha) ridge_reg.fit(X_train, y_train) coefs.append(ridge_reg.coef_) error.append([alpha, np.mean((ridge_reg.predict(X_test) - y_test)**2)]) final_alpha = min(error, key=lambda t: t[1])[0] ridge_reg.set_params(alpha=final_alpha) ridge_reg.fit(X_train, y_train) # Here we select the features based on its coefficient determined from ridge regression in comparison to the highest absolute value of coefficient. # The threshold determines the features considered. abs_coef = abs(ridge_reg.coef_) abs_coef_percent = abs_coef / max( abs_coef ) * 100 # Percentage vis-a-vis the highest absolute coefficient get_features = np.where(abs_coef_percent > thresh)[0] col_df = list(df.columns) list_features = [col_df[i] for i in get_features if col_df[i] != target] return list_features
def ridge_regr(train, test): train_normalized = train / train.std() test_normalized = test / test.std() heat_ridge = Ridge() a = 1e0 heat_ridge.set_params(alpha=a) heat_ridge.fit(train_normalized[['Latitude', 'LoadEntering', 'Longitude']], train_normalized.Capacity) y_predict = heat_ridge.predict( test_normalized[['Latitude', 'LoadEntering', 'Longitude']]) filename = "ridge_result.sav" object_loc = open(filename, 'wb') pickle.dump(heat_ridge, object_loc) object_loc.close() return filename, y_predict
def rr_regress(X, y, a=None, b=None): """ This function returns regression of input data by Ridge regression method. Parameters ---------- X: an array or array-like predictors. It should be scaled by StandardScaler. y: an array or array-like target. It should has compatible dimension with input X. a, b: an array or array-like, optional. another set of data, such as a = X_test, b = y_test. Returns ------- coefs_RR: list. a list of coefficients from RR with different lambdas lambdas_RR: list. a list of lambdas used in this RR error1_RR: list. a list of MSE of prediction from first input set (X, y) error2_RR: list. a list of MSE of prediction from second input set (a, b). Return as None if a and b are not defined. modelRR: modelRR = Ridge(), the LASSO model command """ # RR vs lambda coefs_RR = [] error1_RR = [] error2_RR = [] # Tunning parameter(lambda) lambdas_RR = np.logspace(-4, 8, 200) modelRR = Ridge() # loop over lambda values to determine the best by mse for l in lambdas_RR: modelRR.set_params(alpha=l) modelRR.fit(X, y) coefs_RR.append(modelRR.coef_) error1_RR.append(mean_squared_error(y, modelRR.predict(X))) if a.any() and b.any() is not None: error2_RR.append(mean_squared_error(b, modelRR.predict(a))) else: error2_RR = None return coefs_RR, lambdas_RR, error1_RR, error2_RR, modelRR
def ridge_with_single_search(): data = init_data() X = data[:, 1:] Y = data[:, 0] X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=42) alphas = [0, 0.03, 0.05, 0.1, 0.2, 0.4, 0.5, 0.6, 1, 1.5, 2] _ridge = Ridge() for a in alphas: _ridge.set_params(alpha=a) _ridge.fit(X_train, y_train) test_y_pred = _ridge.predict(X_test) print("alpha is %f,测试集MSE is %f" % (a, mean_squared_error(y_test, test_y_pred)))
def Ridge_Reg(Phi_train, Y_train, Phi_test, Y_test, alphas): reg = Ridge() coefs = [] train_MSE = [] test_MSE = [] for a in alphas: reg.set_params(alpha=a) reg.fit(Phi_train, Y_train) coefs.append(reg.coef_) train_pred = reg.predict(Phi_train) train_MSE.append(mean_squared_error(Y_train, train_pred)) test_pred = reg.predict(Phi_test) test_MSE.append(mean_squared_error(Y_test, test_pred)) return {'coefs': coefs, 'train_MSE': train_MSE, 'test_MSE': test_MSE}
def ridge_cv(k, X, y, alphas): splits = k_split(k, X.shape[0]) results = [] for alpha in alphas: temp_ridges = [] for i, split in enumerate(splits): X_train = X.iloc[split[1]] y_train = y.iloc[split[1]] X_test = X.iloc[split[0]] y_test = y.iloc[split[0]] ridge = Ridge(normalize=True) ridge.set_params(alpha=alpha) ridge.fit(X_train, y_train) predicted_y = ridge.predict(X_test) temp_ridges.append(mean_squared_error(y_true=y_test, y_pred=predicted_y)) results.append(sum(temp_ridges)/len(temp_ridges)) return results
def best_param(tipoReg, alphas): print tipoReg Xm = Xtrain.as_matrix() ym = ytrain.as_matrix() k_fold = cross_validation.KFold(len(Xm),10) best_cv_mse = float("inf") if tipoReg == "Ridge": modelo = Ridge(fit_intercept = True) elif tipoReg == "Lasso": modelo = Lasso(fit_intercept = True) for a in alphas: modelo.set_params(alpha=a) mse_list_k10 = [ MSE(modelo.fit(Xm[train], ym[train]).predict(Xm[val]), ym[val]) for train, val in k_fold] if np.mean(mse_list_k10) < best_cv_mse: best_cv_mse = np.mean(mse_list_k10) best_alpha = a print "BEST PARAMETER=%f, MSE(CV)=%f"%(best_alpha, best_cv_mse)
def best_param(tipoReg, alphas): print tipoReg Xm = Xtrain.as_matrix() ym = ytrain.as_matrix() k_fold = cross_validation.KFold(len(Xm), 10) best_cv_mse = float("inf") if tipoReg == "Ridge": modelo = Ridge(fit_intercept=True) elif tipoReg == "Lasso": modelo = Lasso(fit_intercept=True) for a in alphas: modelo.set_params(alpha=a) mse_list_k10 = [ MSE(modelo.fit(Xm[train], ym[train]).predict(Xm[val]), ym[val]) for train, val in k_fold ] if np.mean(mse_list_k10) < best_cv_mse: best_cv_mse = np.mean(mse_list_k10) best_alpha = a print "BEST PARAMETER=%f, MSE(CV)=%f" % (best_alpha, best_cv_mse)
def get_ridge(X, Y): #running ridge with alpha 0 (MLR) ridge = Ridge() ridge.set_params(alpha=0, normalize=True) ridge.fit(X, Y) #Grid search for ridge alphas_ridge = np.linspace(0, 15, 50) tuned_parameters_r = [{'alpha': alphas_ridge}] n_folds = 5 cv = KFold(n_splits=n_folds, shuffle=True) tune_ridge = GridSearchCV(ridge, tuned_parameters_r, cv=cv, refit=True, return_train_score=True, scoring='neg_mean_squared_error') tune_ridge.fit(X, Y) print(tune_ridge.best_params_) print(np.max(tune_ridge.cv_results_['mean_test_score'])) print(np.min(tune_ridge.cv_results_['mean_test_score'])) ridge_best = tune_ridge.best_estimator_ ridge_best.fit(X, Y) print(ridge_best.score(X, Y)) suffix = str(datetime.datetime.now()) model_filename = 'ridge' + suffix + '.sav' pickle.dump(ridge_best, open(model_filename, 'wb')) csv_filename = 'ridge ' + suffix + '.csv' raw_test, test_IDs = load_test() predict = ridge_best.predict(raw_test) predict = np.exp(predict) predict = pd.DataFrame(predict) predict = pd.concat([test_IDs, predict], axis=1) predict.columns = ['Id', 'SalePrice'] predict.to_csv(csv_filename, index=False)
def test_regressor_modifications(self): regressor = Ridge(alpha=1e-8) pcovr = self.model(mixing=0.5, regressor=regressor) # PCovR regressor matches the original self.assertTrue(regressor.get_params() == pcovr.regressor.get_params()) # PCovR regressor updates its parameters # to match the original regressor regressor.set_params(alpha=1e-6) self.assertTrue(regressor.get_params() == pcovr.regressor.get_params()) # Fitting regressor outside PCovR fits the PCovR regressor regressor.fit(self.X, self.Y) self.assertTrue(hasattr(pcovr.regressor, "coef_")) # PCovR regressor doesn't change after fitting pcovr.fit(self.X, self.Y) regressor.set_params(alpha=1e-4) self.assertTrue(hasattr(pcovr.regressor_, "coef_")) self.assertTrue( regressor.get_params() != pcovr.regressor_.get_params())
nb_runs_cv = len(x) logo = LeaveOneOut() # leave on run out ! cv_index = 0 scores = np.zeros((nb_voxels, nb_runs_cv, nb_alphas)) for train, valid in logo.split(y): y_train = [ y[i] for i in train ] # fmri_runs liste 2D colonne = voxels et chaque row = un t_i x_train = [x[i] for i in train] dm = np.vstack(x_train) fmri = np.vstack(y_train) alpha_index = 0 for alpha_tmp in alphas: # compute the r2 for a given alpha for all the voxel model.set_params(alpha=alpha_tmp) model_fitted = model.fit(dm, fmri) r2 = get_r2_score(model_fitted, y[valid[0]], x[valid[0]]) scores[:, cv_index, alpha_index] = r2 alpha_index += 1 cv_index += 1 best_alphas_indexes = np.argmax(np.mean(scores, axis=1), axis=1) voxel2alpha = np.array([alphas[i] for i in best_alphas_indexes]) # compute best alpha for each voxel and group them by alpha-value alpha2voxel = {key: [] for key in alphas} for index in range(len(voxel2alpha)): alpha2voxel[voxel2alpha[index]].append(index) for alpha in alphas: yaml_path = os.path.join(args.output, 'run_{}_alpha_{}.yml'.format(run, alpha))
# In[56]: #linear regression no regularization reg = linear_model.Ridge(alpha=0) reg = reg.fit(x_nn, y_nn) y_predicted = reg.predict(x_nn) plt.figure() plt.plot(time_axis[30:66], y_predicted, 'r-') plt.plot(x_test, yn_test, 'ko', label="Original Noised Data") plt.show() # In[ ]: #linear regression using regularization alpha = 10000000 clf = Ridge() clf.set_params(alpha=100000000) regression = clf.fit(x_nn, y_nn) y_predicted_2 = regression.predict(x_nn) plt.figure() plt.plot(time_axis[30:66], y_predicted_2, 'r-') plt.plot(x_test, yn_test, 'ko', label="Original Noised Data") plt.show() # # Neural networks # # Class 1 # In[53]: model_class1 = get_model_for_Class(18) model_class1.fit(x=x_nn, y=y_nn, epochs=100, batch_size=1)
X = X.drop('intercept', axis=1) Xtrain = X[istrain] ytrain = y[istrain] names_regressors = ["Lcavol", "Lweight", "Age", "Lbph", "Svi", "Lcp", "Gleason", "Pgg45"] alphas_ = np.logspace(4,-1,base=10) Xm = Xtrain.as_matrix() ym = ytrain.as_matrix() k_fold = cross_validation.KFold(len(Xm),10) best_cv_mse = float("inf") model = Ridge(fit_intercept=True,solver='svd') for a in alphas_: model.set_params(alpha=a) mse_list_k10 = [MSE(model.fit(Xm[train], ym[train]).predict(Xm[val]), ym[val]) for train, val in k_fold] if np.mean(mse_list_k10) < best_cv_mse: best_cv_mse = np.mean(mse_list_k10) best_alpha = a print "RIDGE BEST PARAMETER=%f, MSE(CV)=%f"%(best_alpha,best_cv_mse)
ridge_model.intercept_ # In[23]: # linspace(10,-2,100) 10 ile -2 arasında 100 değer lambdalar = 10**np.linspace(10, -2, 100) * 0.5 lambdalar # In[26]: ridge_model = Ridge() katsayilar = [] for i in lambdalar: ridge_model.set_params(alpha=i) ridge_model.fit(X_train, Y_train) katsayilar.append(ridge_model.coef_) katsayilar # In[27]: ax = plt.gca() ax.plot(lambdalar, katsayilar) # katsayiların hepsini gözlemleyebilmek için rakamları birbirine # yakınlaştıralım deyip düzleştirme yapıyoruz ax.set_xscale("log") # renkler katsayı değerleri # x ekseni lambda değerleri
X['Student'] = X['Student'].map({'Yes': 1, 'No': 0}) X['Married'] = X['Married'].map({'Yes': 1, 'No': 0}) X = X.drop('Ethnicity', axis=1) y = Credit['Balance'].values # X.head() if you'd like to look at the data # first without normalization n_alphas = 200 alphas = np.logspace(-1, 4, n_alphas) clf = Ridge(normalize=False) coef_names = list(X.columns) coefs = [] for a in alphas: clf.set_params(alpha=a, normalize=False) clf.fit(X.values, y) coefs.append(clf.coef_) ax = plt.gca() ax.plot(alphas, coefs) ax.set_xscale('log') plt.xlabel('alpha') plt.ylabel('weights') plt.title('Ridge coefficients without Normalization') plt.axis('tight') plt.legend(coef_names) plt.show() # now with normalization
X = preprocessing.scale(X) X_train, X_test, y_train, y_test = cross_validation.train_test_split( X, Y, test_size=0.2) clf = Ridge() #clf.fit(X_train, y_train) alpha_ridge = [0, 0.1, 1, 10, 100, 1000, 10000, 100000] errors_test = [] errors_train = [] L2Norm_weights = [] weights = [] for i in alpha_ridge: clf.set_params(alpha=i) clf.fit(X_train, y_train) Y_pred = clf.predict(X_test) errors_test.append(sqrt(mean_squared_error(y_test, Y_pred))) Y_pred2 = clf.predict(X_train) errors_train.append(sqrt(mean_squared_error(y_train, Y_pred2))) L2Norm_weights.append((np.linalg.norm(clf.coef_))) weights.append(clf.coef_) weights = array(weights) weights = np.matrix.transpose(weights) #************************ PART (A) AND (B)************************************
n_features = X.shape[1] params = {'max_features':['auto','sqrt','log2']} RF_model = GridSearchCV(RF_est, params) RF_model.fit(X,y) print('Best {}'.format(RF_model.best_params_)) print('Performing grid search on GBR') n_features = X.shape[1] params = {'max_features':['auto','sqrt','log2'], 'max_depth':[2, 3]} GBR_model = GridSearchCV(GBR_est, params) GBR_model.fit(X,y) print('Best {}'.format(GBR_model.best_params_)) else: Lin_model = Lin_est.set_params(alpha=100.0) SVR_model = svr_est.set_params(C=1.0) RF_model = RF_est.set_params(max_features='auto') GBR_model = GBR_est.set_params(max_features='auto', max_depth=3) #%% Specify set of models to test model_set = [('Null',LCM.rand_pick_mod()), ('Lin', Lin_model), ('Lin_SVR',SVR_model), ('GBR',GBR_model), ('RF', RF_model)] # model_set = [('Null',LCM.rand_pick_mod()), # ('Lin', Lin_model), # ('RF', RF_model)]
def Model(train_linear, test_linear): train_linear_fea=train_linear.drop(columns=['SalePrice']) train_linear_tar=train_linear.SalePrice x_train, x_test, y_train, y_test = train_test_split(train_linear_fea, train_linear_tar,test_size=0.2, random_state=0) def evaluate(model, test_features, test_labels,train_features, train_labels): predictions = model.predict(test_features) errors = abs(predictions - test_labels) mape = 100 * np.mean(errors / test_labels) accuracy = 100 - mape print('Model Performance') print('Average Error: {:0.4f} degrees.'.format(np.mean(errors))) print('Accuracy = {:0.2f}%.'.format(accuracy)) print("MSE for train data is: %f" % mean_squared_error(y_train, model.predict(x_train))) print("MSE for validation data is: %f" % mean_squared_error(y_test, model.predict(x_test))) return accuracy real_train_tar=np.expm1(train_linear_tar) """ . Lasso model """ lassocv = LassoCV(alphas = np.logspace(-5, 4, 400), ) lassocv.fit(train_linear_fea, train_linear_tar) lassocv_score = lassocv.score(train_linear_fea, train_linear_tar) lassocv_alpha = lassocv.alpha_ print("Best alpha : ", lassocv_alpha, "Score: ",lassocv_score) start=time.time() lasso =Lasso(normalize = True) lasso.set_params(alpha=lassocv_alpha,max_iter = 10000) lasso.fit(x_train, y_train) end=time.time() mean_squared_error(y_test, lasso.predict(x_test)) coef_lasso=pd.Series(lassocv.coef_, index=x_train.columns).sort_values(ascending =False) evaluate(lasso,x_test,y_test,x_train,y_train) print('Time elapsed: %.4f seconds' % (end-start)) y_lasso_predict=lasso.predict(train_linear_fea) x_line = np.arange(700000) y_line=x_line plt.scatter(real_train_tar,np.expm1(y_lasso_predict)) plt.plot(x_line, y_line, color='r') plt.xlabel('Actual Sale Price') plt.ylabel('Predict Sle Price') test_prediction_lasso=np.expm1(lasso.predict(test_linear)) """ . Ridge model """ ridgecv = RidgeCV(alphas = np.logspace(-5, 4, 400)) ridgecv.fit(x_train, y_train) ridgecv_score = ridgecv.score(x_train, y_train) ridgecv_alpha = ridgecv.alpha_ print("Best alpha : ", ridgecv_alpha, "Score: ",ridgecv_score) coef=pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending =False) start=time.time() ridge =Ridge(normalize = True) ridge.set_params(alpha=ridgecv_alpha,max_iter = 10000) ridge.fit(x_train, y_train) end=time.time() mean_squared_error(y_test, ridge.predict(x_test)) coef_ridge=pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending =False) evaluate(ridge,x_test,y_test,x_train,y_train) print('Time elapsed: %.4f seconds' % (end-start)) y_ridge_predict=ridge.predict(train_linear_fea) x_line = np.arange(700000) y_line=x_line plt.scatter(real_train_tar,np.expm1(y_ridge_predict)) plt.plot(x_line, y_line, color='r') plt.xlabel('Actual Sale Price') plt.ylabel('Predict Sle Price') test_prediction_ridge=np.expm1(ridge.predict(test_linear)) """ . Random Forest """ #train=train.drop(columns=['DateSold']) #test=test.drop(columns=['DateSold']) #X_train=train.drop(columns=['SalePrice']) #Y_train=train['SalePrice'] X_train=train_linear_fea Y_train=train_linear_tar x_train_rf, x_test_rf, y_train_rf, y_test_rf = train_test_split(X_train, Y_train,test_size=0.2, random_state=0) n_estimators = [int(x) for x in np.linspace(start = 100, stop = 2000, num = 20)] max_features = ['auto', 'sqrt'] max_depth = [int(x) for x in np.linspace(10, 110, num = 11)] min_samples_split = [2, 5, 10] min_samples_leaf = [1, 2, 4] bootstrap = [True, False] random_grid = {'n_estimators': n_estimators, 'max_features': max_features, 'max_depth': max_depth, 'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf, 'bootstrap': bootstrap} rf = RandomForestRegressor() # Random search of parameters, using 3 fold cross validation, # search across 100 different combinations, and use all available cores # rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1) rf_random.fit(X_train, Y_train) #rf_random.fit(x_train_rf, y_train_rf) rf_random.best_params_ #Random search allowed us to narrow down the range for each hyperparameter. Now that we know where to concentrate our search, # we can explicitly specify every combination of settings to try. param_grid = { 'bootstrap': [False], 'max_depth': [80, 90, 100, 110,120,130], 'max_features': [2, 3], 'min_samples_leaf': [1,2,3, 4], 'min_samples_split': [2,4,6,8, 10, 12], 'n_estimators': [600,700, 800, 900, 1000] } # Create a based model rf = RandomForestRegressor() # Instantiate the grid search model grid_search = GridSearchCV(estimator = rf, param_grid = param_grid, cv = 3, n_jobs = -1, verbose = 2) #grid_search.fit(x_train, y_train) grid_search.fit(X_train, Y_train) grid_search.best_params_ best_random = grid_search.best_estimator_ start=time.time() best_random.fit(x_train_rf,y_train_rf) end=time.time() evaluate(best_random, x_test_rf, y_test_rf,x_train_rf,y_train_rf) print('Time elapsed: %.4f seconds' % (end-start)) y_rf_predict=best_random.predict(train_linear_fea) x_line = np.arange(700000) y_line=x_line plt.scatter(real_train_tar,np.expm1(y_rf_predict)) plt.plot(x_line, y_line, color='r') plt.xlabel('Actual Sale Price') plt.ylabel('Predict Sle Price') importance_rf = pd.DataFrame({'features':train_linear_fea.columns, 'imp':best_random.feature_importances_}).\ sort_values('imp',ascending=False) importance_top20_rf = importance_rf.iloc[:20,] plt.barh(importance_top20_rf.features, importance_top20_rf.imp) plt.xlabel('Feature Importance') test_prediction_rf=np.expm1(best_random.predict(test_linear)) """ . Xgboost """ learning_rate = [round(float(x), 2) for x in np.linspace(start = .1, stop = .2, num = 11)] # Minimum for sum of weights for observations in a node min_child_weight = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] # Maximum nodes in each tree max_depth = [int(x) for x in np.linspace(1, 10, num = 10)] n_estimators=[int(x) for x in np.linspace(start = 100, stop = 2000, num = 20)] subsample=[0.3, 0.4,0.5,0.6, 0.7] model = xgb.XGBRegressor() random_grid = {'learning_rate': learning_rate, 'max_depth': max_depth, 'min_child_weight': min_child_weight, 'subsample': subsample, 'n_estimators':n_estimators } # Make a RandomizedSearchCV object with correct model and specified hyperparams xgb_random = RandomizedSearchCV(estimator=model, param_distributions=random_grid, n_iter=1000, cv=5, verbose=2, random_state=42, n_jobs=-1) start = time.time() # Fit models xgb_random.fit(X_train, Y_train) xgb_random.best_params_ """ best_params_={'learning_rate': 0.1, 'max_depth': 2, 'min_child_weight': 4, 'n_estimators': 900, 'subsample': 0.5} """ model_xgb = XGBRegressor(**xgb_random.best_params_) #model_xgb = XGBRegressor(**best_params_) start=time.time() model_xgb.fit(x_train_rf,y_train_rf) end=time.time() evaluate(model_xgb, x_test_rf, y_test_rf,x_train_rf,y_train_rf) print('Time elapsed: %.4f seconds' % (end-start)) y_xgb_predict=model_xgb.predict(train_linear_fea) x_line = np.arange(700000) y_line=x_line plt.scatter(real_train_tar,np.expm1(y_xgb_predict)) plt.plot(x_line, y_line, color='r') plt.xlabel('Actual Sale Price') plt.ylabel('Predict Sle Price') importance_xgb = pd.DataFrame({'features':train_linear_fea.columns, 'imp':model_xgb.feature_importances_}).\ sort_values('imp',ascending=False) importance_top20_xgb = importance_xgb.iloc[:20,] plt.barh(importance_top20_xgb.features, importance_top20_xgb.imp) plt.xlabel('Feature Importance') test_prediction_xgb=np.expm1(model_xgb.predict(test_linear)) return(test_prediction_lasso, test_prediction_ridge, test_prediction_rf, test_prediction_xgb,y_lasso_predict, y_ridge_predict, y_rf_predict, y_xgb_predict)
def return_a_reg(reg_name): if reg_name == 'RR': reg = Ridge() reg.set_params(alpha=1) return reg if reg_name == 'KRR': reg = KernelRidge() reg.set_params(kernel='rbf', alpha=1) return reg if reg_name == 'SVR': reg = SVR() reg.set_params(kernel='rbf', C=10) return reg if reg_name == 'KNR': reg = KNeighborsRegressor() reg.set_params(n_neighbors=10) return reg if reg_name == 'GP': reg = gaussian_process.GaussianProcess() reg.set_params(regr='constant', corr='squared_exponential', theta0=1e-2, thetaL=1e-4, thetaU=1e-1) return reg if reg_name == 'DT': reg = tree.DecisionTreeRegressor() reg.set_params(max_depth=8, min_samples_leaf=5, random_state=0) return reg elif reg_name == 'RF': reg = RandomForestRegressor() reg.set_params(n_estimators=100, max_depth=5, min_samples_leaf=5, random_state=0) return reg elif reg_name == 'GBDT': reg = GradientBoostingRegressor() reg.set_params(n_estimators=100, max_depth=5, min_samples_leaf=5, random_state=0) return reg
def set_params(self, C=None): if self.l1: Lasso.set_params(self, alpha=C) else: Ridge.set_params(self, alpha=1.0/(2.0*C)) return self
# nodebox section end clf = Ridge() X, y, w = make_regression(n_samples=10, n_features=10, coef=True, random_state=1, bias=3.5) coefs = [] errors = [] alphas = np.logspace(-6, 6, 200) # Train the model with different regularisation strengths for a in alphas: clf.set_params(alpha=a) clf.fit(X, y) coefs.append(clf.coef_) errors.append(mean_squared_error(clf.coef_, w)) # Display results plt.figure(figsize=(20, 6)) plt.subplot(121) ax = plt.gca() ax.plot(alphas, coefs) ax.set_xscale('log') plt.xlabel('alpha') plt.ylabel('weights') plt.title('Ridge coefficients as a function of the regularization') plt.axis('tight')
clf = Ridge() X, y, w = make_regression(n_samples=10, n_features=10, coef=True, random_state=1, bias=3.5) coefs = [] errors = [] alphas = np.logspace(-6, 6, 200) # Train the model with different regularisation strengths for a in alphas: clf.set_params(alpha=a) clf.fit(X, y) coefs.append(clf.coef_) errors.append(mean_squared_error(clf.coef_, w)) # Display results plt.figure(figsize=(20, 6)) plt.subplot(121) ax = plt.gca() ax.plot(alphas, coefs) ax.set_xscale('log') plt.xlabel('alpha') plt.ylabel('weights') plt.title('Ridge coefficients as a function of the regularization') plt.axis('tight')
X_test_sc = scaler_X.transform(X_test) scaler_y = StandardScaler(with_mean=True, with_std=True) y_train_sc = scaler_y.fit_transform(y_train.reshape(-1, 1)).ravel() y_test_sc = scaler_y.transform(y_test.reshape(-1, 1)).ravel() n_alphas = 50 alphas = np.logspace(-1, 8, n_alphas) ridge = Ridge(fit_intercept=True) kernel_ridge = KernelRidge(kernel='poly', gamma=1, degree=3, coef0=1) test_scores_ridge = [] test_scores_kernel = [] for alpha in alphas: ridge.set_params(alpha=alpha) ridge.fit(X_train_sc, y_train_sc) test_mse = mean_squared_error_scorer(ridge, X_test_sc, y_test_sc) test_scores_ridge.append(test_mse) kernel_ridge.set_params(alpha=alpha) kernel_ridge.fit(X_train_sc, y_train_sc) test_mse = mean_squared_error_scorer(kernel_ridge, X_test_sc, y_test_sc) test_scores_kernel.append(test_mse) poly = PolynomialNetworkRegressor(degree=3, n_components=2, tol=1e-3, warm_start=True, random_state=0) test_scores_poly = []
test = df[df.train_test__train ==0] X_test = test.drop(['SalePrice','train_test__train'], axis=1) alphas = 10**np.linspace(10,-2,100)*0.5 alphas.shape ridge = Ridge(normalize = True) coefs = [] for a in alphas: ridge.set_params(alpha = a) ridge.fit(X_train, y_train) coefs.append(ridge.coef_) np.shape(coefs) ax = plt.gca() ax.plot(alphas, coefs) ax.set_xscale('log') plt.axis('tight') plt.xlabel('alpha') plt.ylabel('weights')
N = X.shape[0] X.insert(X.shape[1], 'intercept', np.ones(N)) y = df_scaled['lpsa'] #Pregunta A X = X.drop('intercept', axis=1) Xtrain = X[istrain] ytrain = y[istrain] names_regressors = [ "Lcavol", "Lweight", "Age", "Lpbh", "Svi", "Lcp", "Gleason", "Pgg45" ] alphas_ = np.logspace(4, -1, base=10) coefs = [] model = Ridge(fit_intercept=True, solver='svd') for a in alphas_: model.set_params(alpha=a) model.fit(Xtrain, ytrain) coefs.append(model.coef_) ax = plt.gca() for y_arr, label in zip(np.squeeze(coefs).T, names_regressors): plt.plot(alphas_, y_arr, label=label) plt.legend() ax.set_xscale('log') ax.set_xlim(ax.get_xlim()[::-1]) #reverse axis plt.xlabel('Alpha') plt.ylabel('Weights') plt.title('Regularization Path RIDGE') plt.axis('tight') plt.legend(loc=2) plt.show()