parser.add_argument('-sy','--start_year', type=int, default=2013) parser.add_argument('-ey','--end_year', type=int, default=2016) m_params = vars(parser.parse_args()) if m_params["start_year"] < m_params["end_year"]: stage_1 = True stage_2 = False else: stage_2 = True stage_1 = False X, y, X_val, y_val, X_val2, y_val2, X_sub = data.load(m_params, stage_1) vars = X.columns print("NCAA: L1 classification...\n") clf = Lasso(alpha=0.1) pred_matrix_val = np.zeros((len(X_val), 100)) pred_matrix_test = np.zeros((len(X_sub), 100)) for i in range(100): # Load data X, y, X_val, y_val, X_val2, y_val2, X_sub = data.load(m_params, stage_1) scaler = StandardScaler() X_sub = scaler.fit_transform(X_sub) if stage_1: X_val = scaler.transform(X_val) X_val2 = scaler.transform(X_val2) X = scaler.fit_transform(X) clf.fit(X,y)
def test_crossfit(self): class Wrapper: def __init__(self, model): self._model = model def fit(self, X, y, Q, W=None): self._model.fit(X, y) return self def predict(self, X, y, Q, W=None): return self._model.predict(X), y - self._model.predict(X), X np.random.seed(123) X = np.random.normal(size=(5000, 3)) y = X[:, 0] + np.random.normal(size=(5000, )) folds = list(KFold(2).split(X, y)) model = Lasso(alpha=0.01) nuisance, model_list, fitted_inds = _crossfit(Wrapper(model), folds, X, y, y, W=y, Z=None) np.testing.assert_allclose( nuisance[0][folds[0][1]], model.fit(X[folds[0][0]], y[folds[0][0]]).predict(X[folds[0][1]])) np.testing.assert_allclose( nuisance[0][folds[0][0]], model.fit(X[folds[0][1]], y[folds[0][1]]).predict(X[folds[0][0]])) coef_ = np.zeros(X.shape[1]) coef_[0] = 1 [ np.testing.assert_allclose(coef_, mdl._model.coef_, rtol=0, atol=0.08) for mdl in model_list ] np.testing.assert_array_equal(fitted_inds, np.arange(X.shape[0])) np.random.seed(123) X = np.random.normal(size=(5000, 3)) y = X[:, 0] + np.random.normal(size=(5000, )) folds = list(KFold(2).split(X, y)) model = Lasso(alpha=0.01) nuisance, model_list, fitted_inds = _crossfit(Wrapper(model), folds, X, y, None, W=y, Z=None) np.testing.assert_allclose( nuisance[0][folds[0][1]], model.fit(X[folds[0][0]], y[folds[0][0]]).predict(X[folds[0][1]])) np.testing.assert_allclose( nuisance[0][folds[0][0]], model.fit(X[folds[0][1]], y[folds[0][1]]).predict(X[folds[0][0]])) coef_ = np.zeros(X.shape[1]) coef_[0] = 1 [ np.testing.assert_allclose(coef_, mdl._model.coef_, rtol=0, atol=0.08) for mdl in model_list ] np.testing.assert_array_equal(fitted_inds, np.arange(X.shape[0])) np.random.seed(123) X = np.random.normal(size=(5000, 3)) y = X[:, 0] + np.random.normal(size=(5000, )) folds = list(KFold(2).split(X, y)) model = Lasso(alpha=0.01) nuisance, model_list, fitted_inds = _crossfit(Wrapper(model), folds, X, y, None, W=None, Z=None) np.testing.assert_allclose( nuisance[0][folds[0][1]], model.fit(X[folds[0][0]], y[folds[0][0]]).predict(X[folds[0][1]])) np.testing.assert_allclose( nuisance[0][folds[0][0]], model.fit(X[folds[0][1]], y[folds[0][1]]).predict(X[folds[0][0]])) coef_ = np.zeros(X.shape[1]) coef_[0] = 1 [ np.testing.assert_allclose(coef_, mdl._model.coef_, rtol=0, atol=0.08) for mdl in model_list ] np.testing.assert_array_equal(fitted_inds, np.arange(X.shape[0])) class Wrapper: def __init__(self, model): self._model = model def fit(self, X, y, W=None): self._model.fit(X, y) return self def predict(self, X, y, W=None): return self._model.predict(X), y - self._model.predict(X), X np.random.seed(123) X = np.random.normal(size=(5000, 3)) y = X[:, 0] + np.random.normal(size=(5000, )) folds = [(np.arange(X.shape[0] // 2), np.arange(X.shape[0] // 2, X.shape[0])), (np.arange(X.shape[0] // 2), np.arange(X.shape[0] // 2, X.shape[0]))] model = Lasso(alpha=0.01) with pytest.raises(AttributeError) as e_info: nuisance, model_list, fitted_inds = _crossfit(Wrapper(model), folds, X, y, W=y, Z=None) np.random.seed(123) X = np.random.normal(size=(5000, 3)) y = X[:, 0] + np.random.normal(size=(5000, )) folds = [(np.arange(X.shape[0]), np.arange(X.shape[0]))] model = Lasso(alpha=0.01) with pytest.raises(AttributeError) as e_info: nuisance, model_list, fitted_inds = _crossfit(Wrapper(model), folds, X, y, W=y, Z=None)
def solve(self, fraction_evaluated, dim): eyAdj = self.linkfv(self.ey[:, dim]) - self.link.f(self.fnull[dim]) s = np.sum(self.maskMatrix, 1) # do feature selection if we have not well enumerated the space nonzero_inds = np.arange(self.M) log.debug("fraction_evaluated = {0}".format(fraction_evaluated)) if (self.l1_reg not in [ "auto", False, 0 ]) or (fraction_evaluated < 0.2 and self.l1_reg == "auto"): w_aug = np.hstack( (self.kernelWeights * (self.M - s), self.kernelWeights * s)) log.info("np.sum(w_aug) = {0}".format(np.sum(w_aug))) log.info("np.sum(self.kernelWeights) = {0}".format( np.sum(self.kernelWeights))) w_sqrt_aug = np.sqrt(w_aug) eyAdj_aug = np.hstack( (eyAdj, eyAdj - (self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim])))) eyAdj_aug *= w_sqrt_aug mask_aug = np.transpose( w_sqrt_aug * np.transpose(np.vstack( (self.maskMatrix, self.maskMatrix - 1)))) var_norms = np.array([ np.linalg.norm(mask_aug[:, i]) for i in range(mask_aug.shape[1]) ]) if self.l1_reg == "auto": model = LassoLarsIC(criterion="aic") elif self.l1_reg == "bic" or self.l1_reg == "aic": model = LassoLarsIC(criterion=self.l1_reg) else: model = Lasso(alpha=self.l1_reg) model.fit(mask_aug, eyAdj_aug) nonzero_inds = np.nonzero(model.coef_)[0] if len(nonzero_inds) == 0: return np.zeros(self.M), np.ones(self.M) # eliminate one variable with the constraint that all features sum to the output eyAdj2 = eyAdj - self.maskMatrix[:, nonzero_inds[-1]] * ( self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim])) etmp = np.transpose( np.transpose(self.maskMatrix[:, nonzero_inds[:-1]]) - self.maskMatrix[:, nonzero_inds[-1]]) log.debug("etmp[:4,:] {0}".format(etmp[:4, :])) # solve a weighted least squares equation to estimate phi tmp = np.transpose( np.transpose(etmp) * np.transpose(self.kernelWeights)) tmp2 = np.linalg.inv(np.dot(np.transpose(tmp), etmp)) w = np.dot(tmp2, np.dot(np.transpose(tmp), eyAdj2)) log.debug("np.sum(w) = {0}".format(np.sum(w))) log.debug("self.link(self.fx) - self.link(self.fnull) = {0}".format( self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim]))) log.debug("self.fx = {0}".format(self.fx[dim])) log.debug("self.link(self.fx) = {0}".format(self.link.f(self.fx[dim]))) log.debug("self.fnull = {0}".format(self.fnull[dim])) log.debug("self.link(self.fnull) = {0}".format( self.link.f(self.fnull[dim]))) phi = np.zeros(self.M) phi[nonzero_inds[:-1]] = w phi[nonzero_inds[-1]] = (self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim])) - sum(w) log.info("phi = {0}".format(phi)) # clean up any rounding errors for i in range(self.M): if np.abs(phi[i]) < 1e-10: phi[i] = 0 return phi, np.ones(len(phi))
predict_r = r.predict(test) mse = mean_squared_error(test_label, predict_r) r_score = np.sqrt(mse) r_score #cross_val_ridge r = Ridge(alpha=0.05, solver='cholesky') score = cross_val_score(r, train, train_label, cv=10, scoring='neg_mean_squared_error') r_score_cross = np.sqrt(-score) np.mean(r_score_cross), np.std(r_score_cross) from sklearn.linear_model import Lasso l = Lasso(alpha=0.01) l.fit(train, train_label) predict_l = l.predict(test) mse = mean_squared_error(test_label, predict_l) l_score = np.sqrt(mse) l_score #cross_val_lasso l = Lasso(alpha=0.01) score = cross_val_score(l, train, train_label, cv=10, scoring='neg_mean_squared_error') l_score_cross = np.sqrt(-score) np.mean(l_score_cross), np.std(l_score_cross)
def Lasso_new(self, X, y): # parameter tuning alphaarr = [ 0.07, 0.009, 0.008, 0.001, 0.09, 0.08, 0.985, 0.085, 0.05, 0.01, 0.1 ] print("++Sumit+ code-- LAsso") alphaarr.sort() print('alphaarr', alphaarr) ##LASSO CODE '''scaler = StandardScaler() scaler.fit(X) X = scaler.transform(X)''' # , alpha=float(alpha) best_result = [] max_result = [] min_result = [] kfold = model_selection.KFold(n_splits=10) for alpha in alphaarr: clf = SGDClassifier(loss="log", penalty="l1", alpha=float(alpha)) ress = clf.fit(X, y) model = fs.SelectFromModel(ress, prefit=True) X_new = model.transform(X) if X_new.shape[1] > 2: cv_results = model_selection.cross_val_score( SGDClassifier(loss="log", penalty="l1", alpha=float(alpha)), X_new, y, cv=kfold, scoring='accuracy') ## check if train data can be scaled # print("cv_results",cv_results.mean(),'alpha --',alpha) best_result.append(cv_results.mean()) max_result.append(cv_results.max()) min_result.append(cv_results.min()) bob = best_result.index(max(best_result)) print('bob--', bob, 'best_result-', best_result[bob], 'alpha-', alphaarr[bob]) print('max--', max_result[bob]) print('max--', min_result[bob]) clf = SGDClassifier(loss="log", penalty="l1", alpha=float(alphaarr[bob])) ress = clf.fit(X, y) model = fs.SelectFromModel(ress, prefit=True) X_new = model.transform(X) print(X_new.shape) regr = Lasso(alpha=alphaarr[bob]) regr.fit(X, y) y_pred = regr.predict(X) print("Lasso score on training set: ", numpy.sqrt(mean_squared_error(y, y_pred))) #ret.extend([regr.coef_]) #coef_list=list(regr.sparse_coef_.data) #coefic=pd.DataFrame(regr.sparse_coef_.toarray()) print('sparse coef -', regr.sparse_coef_) coref_arr = numpy.array(regr.sparse_coef_.toarray()) print(len(coref_arr[0])) proce = [] a = { x: coref_arr[0][x] for x in range(len(coref_arr[0])) if coref_arr[0][x] != 0.0 } #print(a[:3]) #print('intercept -',regr.intercept_) #print(type(coefic)) #print(coefic(1,1)) sorta = list(sorted(a, key=a.__getitem__, reverse=True)) print(sorta) print(sorta[1]) print(a.values()[1])
ridge.fit(X_train, y_train) train_score_list.append(ridge.score(X_train, y_train)) test_score_list.append(ridge.score(X_test, y_test)) plt.plot(x_range, train_score_list, c='g', label='Train Score') plt.plot(x_range, test_score_list, c='b', label='Test Score') plt.xscale('log') plt.legend(loc=3) plt.xlabel(r'$\alpha$') #LASSO x_range = [0.01, 0.1, 1, 10] train_score_list = [] test_score_list = [] for alpha in x_range: lasso = Lasso(alpha) lasso.fit(X_train, y_train) train_score_list.append(lasso.score(X_train, y_train)) test_score_list.append(lasso.score(X_test, y_test)) #KNEIGHBORS train_score_array = [] test_score_array = [] for k in range(1, 20): knn = KNeighborsClassifier(k) knn.fit(X_train, y_train) train_score_array.append(knn.score(X_train, y_train)) test_score_array.append(knn.score(X_test, y_test))
draw_dependency_ratio(dataframe=df, list_of_features=important_features, target_feature="Chance of Admit") x_full = pd.DataFrame(np.c_[df[important_features]], columns=important_features) y_full = df['Chance of Admit'] x_train, x_test, y_train, y_test = train_test_split(x_full, y_full, test_size=0.2, random_state=5) print("The Linear Regression") lin_model = LinearRegression() lin_model.fit(x_train, y_train) print_model_stat(lin_model, x_train, y_train, x_test, y_test, False) print("\n") print("The Linear Ridge Regression") lin_model_ridge = Ridge() lin_model_ridge.fit(x_train, y_train) print_model_stat(lin_model_ridge, x_train, y_train, x_test, y_test, False) print("\n") print("The Linear lasso Regression") lin_model_lasso = Lasso() lin_model_lasso.fit(x_train, y_train) print_model_stat(lin_model_lasso, x_train, y_train, x_test, y_test, False) print("\n")
f.fit(xtrain,ytrain) f.intercept_,f.coef_ f.score(xtrain,ytrain) f.score(xtest,ytest) # ridge regression from sklearn.linear_model import Ridge f = Ridge(alpha=0.5) f.fit(xtrain,ytrain) f.intercept_,f.coef_ f.score(xtrain,ytrain) f.score(xtest,ytest) # lasso regression from sklearn.linear_model import Lasso f = Lasso(alpha=0.5) f.fit(xtrain,ytrain) f.intercept_,f.coef_ f.score(xtrain,ytrain) f.score(xtest,ytest) # Elastic Net regression from sklearn.linear_model import ElasticNet f = ElasticNet(alpha=0.1,l1_ratio=0.5) f.fit(xtrain,ytrain) f.intercept_,f.coef_ f.score(xtrain,ytrain) f.score(xtest,ytest) # select parameter using cross-validation np.random.seed(0)
plt.show() y_predict = y_predict_ridge RMSE = float(format(np.sqrt(mean_squared_error(y_test, y_predict)), '.3f')) MSE = mean_squared_error(y_test, y_predict) MAE = mean_absolute_error(y_test, y_predict) r2 = r2_score(y_test, y_predict) adj_r2 = 1 - (1 - r2) * (n - 1) / (n - k - 1) print('RIDGE:', '\nRMSE =', RMSE, '\nMSE =', MSE, '\nMAE =', MAE, '\nR2 =', r2, '\nAdjusted R2 =', adj_r2) """# LASSO REGRESSION""" from sklearn.linear_model import Lasso regressor_lasso = Lasso(alpha=500) regressor_lasso.fit(X_train, y_train) print('Linear Model Coefficient (m): ', regressor_lasso.coef_) print('Linear Model Coefficient (b): ', regressor_lasso.intercept_) y_predict_lasso = regressor_lasso.predict(X_test) y_predict_lasso plt.plot(y_test, y_predict_lasso, "^", color='brown') plt.xlim(0, 3000000) plt.ylim(0, 3000000) plt.xlabel("Model Predictions") plt.ylabel("True Value (ground Truth)") plt.title('Lasso Regression Predictions') plt.show()
param_grid=param_grid, splits=splits, repeats=repeats) cv_score.name = model score_models = score_models.append(cv_score) plt.figure() plt.errorbar(alph_range, abs(grid_results['mean_test_score']), abs(grid_results['std_test_score']) / np.sqrt(splits * repeats)) plt.xlabel('alpha') plt.ylabel('score') model = 'Lasso' opt_models[model] = Lasso() alph_range = np.arange(1e-4, 1e-3, 4e-5) param_grid = {'alpha': alph_range} opt_models[model], cv_score, grid_results = train_model(opt_models[model], param_grid=param_grid, splits=splits, repeats=repeats) cv_score.name = model score_models = score_models.append(cv_score) plt.figure() plt.errorbar(alph_range, abs(grid_results['mean_test_score']), abs(grid_results['std_test_score']) / np.sqrt(splits * repeats)) plt.xlabel('alpha')
def make_estimator(self, params): representation, precompute = params estimator = Lasso(precompute=precompute, alpha=0.001, random_state=0) return estimator
RSS = sum((test_Y - lr_predictions) ** 2) print('Linear Regression\nTest RSS: ' + str(RSS)) cv_risk = math.sqrt(sum(abs(cross_val_score(linear_regression_model, train_X, train_Y, scoring='mean_squared_error', cv=10))) / 10) print('10-fold CV with RMSE: ' + str(cv_risk)) plt.plot(np.linspace(0, 40, 40), lr_predictions, 'r', label="predictions") plt.plot(np.linspace(0, 40, 40), test_Y, label="real values") plt.legend(loc='lower right') plt.title(title) plt.show() # 2. Using Lasso Regression on Data # Optimization Objective: (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1 print('\nLasso Model:') alpha = 0.1 lasso_model = Lasso(alpha=alpha) lasso_line = lasso_model.fit(train_X, train_Y) title = 'Lasso with Alpha = ' + str(alpha) lasso_predictions = lasso_line.predict(test_X) RSS = sum((test_Y - lasso_predictions) ** 2) print('Lasso with Lambda 0.1\nTest RSS: ' + str(RSS)) cv_risk = math.sqrt(sum(abs(cross_val_score(lasso_model, train_X, train_Y, scoring='mean_squared_error', cv=10))) / 10) print('10-fold CV with RMSE: ' + str(cv_risk)) plt.plot(np.linspace(0, 40, 40), lasso_predictions, 'r', label="predictions") plt.plot(np.linspace(0, 40, 40), test_Y, label="real values") plt.legend(loc='lower right') plt.title(title) plt.show() # Testing for different alphas alphas = [0.0001, 0.001, 0.01, 0.1, 1, 10]
from sklearn.model_selection import train_test_split X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0) from sklearn.linear_model import SGDRegressor from sklearn.linear_model import LinearRegression, Ridge, Lasso from sklearn.svm import SVR from sklearn.tree import DecisionTreeRegressor SGD = SGDRegressor(max_iter=10000, random_state=0) LR = LinearRegression() RI = Ridge(alpha=1) LA = Lasso() SVM = SVR(degree=2, kernel='rbf', C=1e3, gamma=0.01) DT = DecisionTreeRegressor(random_state=0) catagory_list = list(X.columns) variable_combination = [] import itertools for i in range(29, len(catagory_list) + 1): for subset in itertools.combinations(catagory_list, i): variable_combination.append(list(subset)) print('combination 완성 : %d' % len(variable_combination)) SGD_acc = []
lin_mse = mean_squared_error(y_train, consum_predictions) lin_rmse = np.sqrt(lin_mse) print('linear_train_rmse', lin_rmse) #model might be underfitting from sklearn.model_selection import cross_val_score scores = cross_val_score(lin_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=10) lin_rmse_scores = np.sqrt(-scores) def explain_scores(scores): print("Scores:", scores) print("Mean:", scores.mean()) print("Standard deviation:", scores.std()) explain_scores(lin_rmse_scores) from sklearn.linear_model import Lasso regLasso1 = Lasso(fit_intercept=False,normalize=False) print(regLasso1) regLasso1.fit(X_train, y_train) print(regLasso1.coef_) my_alphas = np.array([0.001,0.01,0.02,0.025,0.05,0.1,0.25,0.5,0.8,1.0]) from sklearn.linear_model import lasso_path alpha_for_path, coefs_lasso, _ = lasso_path(X_train, y_train ,alphas=my_alphas) print(coefs_lasso.shape) import matplotlib.cm as cm couleurs = cm.rainbow(numpy.linspace(0,1,16))
def lasso_estimator(alpha=1.0): model = Pipeline([('normalize', StandardScaler()), ('lasso', Lasso(alpha=alpha))]) return model
r_y = real_table['sales'] #c_x c_r_x = compare_table.iloc[:,3:-3] c_r_y = compare_table['sales'] ####################################################Ridge 모델##################################### ridge_m = Ridge() ridge_params = { 'max_iter':[3000], 'alpha': [0.01, 0.1, 1, 2, 3, 4, 10, 30, 100, 200, 300, 400, 800, 900, 1000]} rmsle_scorer = metrics.make_scorer(rmsle, greater_is_better = False) grid_ridge_m = GridSearchCV( ridge_m, ridge_params, cv = 5) grid_ridge_m.fit(r_x,r_y) print(grid_ridge_m.score(c_r_x, c_r_y)) ####################################################Lasso 모델##################################### lasso_m = Lasso() alpha = 1/np.array([0.01, 0.1, 1, 2, 3, 4, 10, 30, 100, 200, 300, 400, 800, 900, 1000]) lasso_params = { 'max_iter':[3000], 'alpha': alpha} grid_lasso_m = GridSearchCV( lasso_m, lasso_params, cv = 5) grid_lasso_m.fit(r_x,r_y) print(grid_lasso_m.score(c_r_x, c_r_y)) pred_lasso = grid_lasso_m.predict(c_r_x) print(pred_lasso) print(len(pred_lasso)) ####################################################svr 모델##################################### from sklearn.svm import SVR svr = SVR(C = 1.0, epsilon = 0.2) svr.fit(r_x,r_y) svr.score(c_r_x, c_r_y)
def lasso_model(): return Lasso(alpha=0.05)
ridge_regressor=GridSearchCV(ridge, parameters,scoring='neg_mean_squared_error', cv=5) ridge_regressor.fit(Xs,y) print(ridge_regressor.best_params_) print(ridge_regressor.best_score_) # pick one with lwest score and lowest MSE # Lasso Regression from sklearn.linear_model import Lasso lasso = Lasso() parameters = {'alpha': [1e-15, 1e-10, 1e-8, 1e-4, 1e-3, 1e-2,1,5,10,20]} lasso_regressor=GridSearchCV(lasso, parameters,scoring='neg_mean_squared_error', cv=5) lasso_regressor.fit(Xs,y) print(lasso_regressor.best_params_) print(lasso_regressor.best_score_) ######## END OF CODE ######## # Darren Nicol- Management Science Project for MSc in Financial Technology #
#We use the cross_val_score function of Sklearn. However this function has not a shuffle attribut, # we add then one line of code, in order to shuffle the dataset prior to cross-validation #Validation function n_folds = 5 def rmsle_cv(model): kf = KFold(n_folds, shuffle=True, random_state=42).get_n_splits(train.values) rmse = np.sqrt(-cross_val_score( model, train.values, y_train, scoring="neg_mean_squared_error", cv=kf)) return (rmse) lasso = make_pipeline(RobustScaler(), Lasso(alpha=0.0005, random_state=1)) ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3)) ##KRR = KernelRidge(alpha=0.6, kernel='sigmoid', gamma=1, coef0=2.5) RFR = RandomForestRegressor(n_estimators=600) GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5) model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468, learning_rate=0.05,
print ('Coefficients:',ridge.coef_) print ('Intercept:',ridge.intercept_) print('MSE train: %.3f, test: %.3f' % (mean_squared_error(y_train, y_train_pred),mean_squared_error(y_test, y_test_pred))) print('R^2 train: %.3f, test: %.3f' % (r2_score(y_train, y_train_pred),r2_score(y_test, y_test_pred))) from sklearn.linear_model import RidgeCV ridgecv = RidgeCV(alphas=[0.00001,0.0001,0.001,0.01, 0.1, 0.5, 1, 10]) ridgecv.fit(X_std, y_std) print ('-----------The alpha gives the best performing model:',ridgecv.alpha_) #LASSO Regression from sklearn.linear_model import Lasso for l in [0.00001,0.0001,0.001,0.01, 0.1, 0.5, 1, 10]: lasso = Lasso(alpha=l) lasso.fit(X_train_std, y_train) y_train_pred = lasso.predict(X_train_std) y_test_pred = lasso.predict(X_test_std) y_train_pred=y_train_pred.reshape(-1,1) y_test_pred=y_test_pred.reshape(-1,1) print('===============','Alpha=',l,'===============') #print('Training accuracy:', ridge.score(X_train_std, y_train),'Test accuracy:', ridge.score(X_test_std, y_test)) plt.scatter(y_train_pred, y_train_pred - y_train,c='steelblue', marker='o', edgecolor='white',label='Training data') plt.scatter(y_test_pred, y_test_pred - y_test,c='limegreen', marker='s', edgecolor='white',label='Test data') plt.xlabel('Predicted values') plt.ylabel('Residuals') plt.legend(loc='upper left') plt.hlines(y=0, xmin=-2, xmax=2, color='black', lw=2) plt.xlim([-2, 2])
def __init__(self, train, y_train, test, n_folds): self.train = train self.y_train = y_train self.test = test self.n_folds = n_folds #Validation function #LASSO Regression self.lasso = make_pipeline(RobustScaler(), Lasso(alpha=0.0005, random_state=1)) #Elastic Net Regression self.ENet = make_pipeline( RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3)) #Kernel Ridge Regression self.KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5) #Gradient Boosting Regression self.Boost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5) #XGBoost self.model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468, learning_rate=0.05, max_depth=3, min_child_weight=1.7817, n_estimators=800, reg_alpha=0.4640, reg_lambda=0.8571, subsample=0.5213, silent=1, random_state=7, nthread=-1) #LightGBM self.model_lgb = lgb.LGBMRegressor(objective='regression', num_leaves=5, learning_rate=0.05, n_estimators=720, max_bin=55, bagging_fraction=0.8, bagging_freq=5, feature_fraction=0.2319, feature_fraction_seed=9, bagging_seed=9, min_data_in_leaf=6, min_sum_hessian_in_leaf=11) self.average_model = AveragingModels(models=(self.ENet, self.Boost, self.KRR)) self.stack_model = StackingAveragedModels( base_models=([self.ENet, self.Boost, self.KRR]), meta_model=self.lasso)
# Every coefficient looks pretty stable, which mean that different Ridge model # put almost the same weight to the same feature. # %% [markdown] # ### Linear models with sparse coefficients (Lasso) # %% [markdown] # In it important to keep in mind that the associations extracted depend # on the model. To illustrate this point we consider a Lasso model, that # performs feature selection with a L1 penalty. Let us fit a Lasso model # with a strong regularization parameters `alpha` # %% from sklearn.linear_model import Lasso model = make_pipeline(StandardScaler(), Lasso(alpha=.015)) model.fit(X_train, y_train) print(f'model score on training data: {model.score(X_train, y_train)}') print(f'model score on testing data: {model.score(X_test, y_test)}') # %% coefs = pd.DataFrame( model[1].coef_, columns=['Coefficients'], index=X_train.columns ) coefs.plot(kind='barh', figsize=(9, 7)) plt.title('Lasso model, strong regularization') plt.axvline(x=0, color='.5')
def simulate_SR_code( N, # Pop size d, # Num defectives est_d = 1, # Estimated prevalence (%) m = None, # Number of layers b = None, # Inverse of size of batches (1 = max size, 2 = half max) verbose = False, alpha=0.001, pt=0.1): # Arithmetic tables for the finite field of order 16 FFsum =[[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15], [1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14], [2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13], [3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12], [4,5,6,7,0,1,2,3,12,13,14,15,8,9,10,11], [5,4,7,6,1,0,3,2,13,12,15,14,9,8,11,10], [6,7,4,5,2,3,0,1,14,15,12,13,10,11,8,9], [7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8], [8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7], [9,8,11,10,13,12,15,14,1,0,3,2,5,4,7,6], [10,11,8,9,14,15,12,13,2,3,0,1,6,7,4,5], [11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4], [12,13,14,15,8,9,10,11,4,5,6,7,0,1,2,3], [13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2], [14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1], [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ] FFmul =[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15], [0,2,4,6,8,10,12,14,3,1,7,5,11,9,15,13], [0,3,6,5,12,15,10,9,11,8,13,14,7,4,1,2], [0,4,8,12,3,7,11,15,6,2,14,10,5,1,13,9], [0,5,10,15,7,2,13,8,14,11,4,1,9,12,3,6], [0,6,12,10,11,13,7,1,5,3,9,15,14,8,2,4], [0,7,14,9,15,8,1,6,13,10,3,4,2,5,12,11], [0,8,3,11,6,14,5,13,12,4,15,7,10,2,9,1], [0,9,1,8,2,11,3,10,4,13,5,12,6,15,7,14], [0,10,7,13,14,4,9,3,15,5,8,2,1,11,6,12], [0,11,5,14,10,1,15,4,7,12,2,9,13,6,8,3], [0,12,11,7,5,9,14,2,10,6,1,13,15,3,4,8], [0,13,9,4,1,12,8,5,2,15,11,6,3,14,10,7], [0,14,15,1,13,3,2,12,9,7,6,8,4,10,11,5], [0,15,13,2,9,6,4,11,1,14,12,3,8,7,5,10] ] def power(base,exp): res = 1 for i in range(exp): res = FFmul[res][base] return res def evaluate(poly, x): # Evaluate polynomial with coeficients in poly at x res = 0 for exp,coef in enumerate(poly): res = FFsum[res][FFmul[coef][power(x,exp)]] return res def printv(x): if verbose: print(x) # Empirically found good parameters good_params = { 0.1: {'m':4, 'b':1}, 1: {'m':5, 'b':1}, 5: {'m':6,'b':2} } #===== Start ======== start = time.time() true_pos = 0 false_pos = 0 false_neg = 0 num_tests = 0 pool_size = [] if m is None: m = good_params[est_d]['m'] if b is None: b = good_params[est_d]['b'] # Randomize the infected and store them as an N-dimensional binary vector x x = [0 for i in range(N)] D = 0 while D < d: num = random.randint(0,N-1) if x[num] == 0: x[num] = 1 D += 1 # We divide the population in batches of size at most 512. This is necessary so # that pool sizes won't exceed 32, our estimated upper bound batch_size = 16*32 iter = 0 Ncount = 0 while(Ncount < N): printv(f"Beginning iteration {iter}") n = ceil(batch_size/b if N-Ncount > batch_size/b else N-Ncount) Ncount += n # n = size of current batch printv(f"n = {n}") # TODO test different values of q (Note that this requires changing # the finite field q = 16 # Choose minimum k that verifies the constraint # n < q^k => log_q(n) < k k = ceil(log(n)/log(q)) # Assign polynomials to x. Count carries the current polynomial count = [0 for i in range(k)] # polys[i] is the polynomial of individual i polys = [] for i in range(n): polys.append(count.copy()) acc = 1 for j in range(k): count[j] = count[j] + acc acc = 0 if count[j] >= q: count[j] = count[j] % q acc = 1 #print(polys[i]) # Select the current batch x_act = x[Ncount-n:Ncount] # Layers M = [] for j in range(m): M.append([[0 for i in range(n)] for i in range(q)]) # Layer j for i in range(n): # Column i l = evaluate(polys[i],j) M[j][l][i] = 1 M_flat = [row for layer in M for row in layer] image = [[255 if item == 0 else 1 for item in row] for row in M_flat] png.from_array(image, 'L').save(f"designs/design{iter}.png") printv(f"Design saved to 'design{iter}.png'. Coding...") y = [reduce(lambda a,b: min(a + b[0]*b[1],1), zip(row,x_act), 0) for row in M_flat] #print(y) printv("Decoding...") # TODO better decoder? model = Lasso(alpha=alpha, positive=True) model.fit(sp.coo_matrix(M_flat),y) # Prediction vector. If a value in x_pred is > pt, we take it as positive x_pred = model.coef_ true_positives = sum([(1 if real==1 and pred > pt else 0) for real,pred in zip(x_act,x_pred)]) true_pos += true_positives false_positives = sum([(1 if real==0 and pred > pt else 0) for real,pred in zip(x_act,x_pred)]) false_pos += false_positives false_negatives = sum([1 if real==1 and pred < pt else 0 for real,pred in zip(x_act,x_pred)]) false_neg += false_negatives num_tests += q*m pool_size.append(ceil(n/q)) iter += 1 printv(f"Done in {model.n_iter_} iterations") ttime = time.time() - start printv(f"Done in {ttime} seconds") printv(f"Tamaño de la población: {N}, número de infectados: {d}") printv(f"Número de positivos: {true_pos + false_pos}") printv(f"True positives: {true_pos}") printv(f"False positives: {false_pos}") printv(f"False negatives: {false_neg}") printv(f"Número de tests realizados: {num_tests}") printv(f"Mean pool size: {mean(pool_size)}") printv(f"Max pool size: {max(pool_size)}") return {'tp':true_pos, 'fp':false_pos, 'fn':false_neg, 'nt':num_tests, 'ps':max(pool_size), 'st':true_pos/(true_pos+false_neg)}
import numpy as np # Regressions from sklearn.linear_model import LinearRegression, Ridge, Lasso, SGDClassifier, SGDRegressor from sklearn.svm import SVR from sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor, RandomForestRegressor # Classifiers from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier models_regression = [ LinearRegression(), Ridge(random_state=42, max_iter=100), Lasso(random_state=42, max_iter=100), SVR(gamma='scale'), AdaBoostRegressor(random_state=42, n_estimators=10), GradientBoostingRegressor(random_state=42, max_depth=3, n_estimators=10), RandomForestRegressor(n_estimators=10, random_state=42, max_depth=3) ] models_classification = [ LogisticRegression(solver='lbfgs', max_iter=100, random_state=42), SVC(gamma='scale', max_iter=100, probability=True), AdaBoostClassifier(random_state=42, n_estimators=10), GradientBoostingClassifier(random_state=42, max_depth=3, n_estimators=10), RandomForestClassifier(n_estimators=10, random_state=42, max_depth=3) ]
def runLASSO(self, X, y): #parameter tuning alphaarr = [ str(1e-20), str(1e-19), str(1e-18), str(1e-17), str(1e-16), str(1e-15), str(1e-14), str(1e-13), str(1e-12), str(1e-11), str(1e-10), str(1e-9), str(1e-8), str(1e-7), str(1e-6), str(1e-5), str(1e-4), str(1e-3), str(1e-2), str(1e-1), str(1e0), str(1e1), str(1e2), str(1e3), str(1e4), str(1e5) ] #coef = numpy.zeros((alpha.len())) #for i in range(0, alpha.len()): # clf = SGDClassifier(loss="log", penalty="l1", alpha=alpha[i]) # clf.fit(X, y) # coef[i] = clf.coef_ #logical programming #for i in range(0, alpha.len()): # for k in range(0, coef[i].len()): # if coef[i][k] == -1: # coef[i][k] = 0 ##X=X.tolist() ##X = [[int(float(j)) for j in i] for i in X] ##y = [int(float(i)) for i in y] print("++Sumit+ code-- LAsso") ##LASSO CODE regr = Lasso(alpha=0.15) regr.fit(X, y) y_pred = regr.predict(X) print("Lasso score on training set: ", numpy.sqrt(mean_squared_error(y, y_pred))) rss = sum((y_pred - y)**2) ret = [rss] ret.extend([regr.intercept_]) #ret.extend([regr.coef_]) print('sparse coef -', regr.sparse_coef_) print('intercept -', regr.intercept_) #foldNo = 10 ## chose which fold have best accuracy results = [] #skf = StratifiedKFold(y, n_folds=foldNo) #print("skf--",skf) #print("X =", X, "\nY = ", y) '''for train_index, test_index in skf: X_train, X_test = numpy.array(X[train_index]), numpy.array(X[test_index]) print("XTrain=", X_train, "\nXTest=", X_test) y=numpy.array(y) y_train = numpy.array(y[train_index]) y_test = numpy.array(y[test_index]) print("\ny_train=", y_train, "\ny_test=", y_test) clf = SGDClassifier(loss="log", penalty="l1", alpha=0.1) clf.fit(X_train, y_train) print('clf in loop is', clf) y_predSGD=clf.predict(X) results.append(y_predSGD) ''' scaler = StandardScaler() scaler.fit(X) X = scaler.transform(X) #, alpha=float(alpha) best_result = [] kfold = model_selection.KFold(n_splits=10) for alpha in alphaarr: cv_results = model_selection.cross_val_score(SGDClassifier( loss="log", penalty="l1", alpha=float(alpha)), X, y, cv=kfold, scoring='accuracy') ## check if train data can be scaled #print("cv_results",cv_results.mean(),'alpha --',alpha) best_result.append(cv_results.mean()) bob = best_result.index(max(best_result)) print('bob--', bob, 'best_result-', best_result[bob], 'alpha-', alphaarr[bob]) ##y_predicted = StratifiedKFold(X, n_folds=foldNo, shuffle=False, random_state=None) #accuracyCalculation(y_predicted, Go.lowclass, instOrder) #print("results---",results) #use parameters with highest overall accuracy #clf = SGDClassifier(loss="log", penalty="l1", alpha=float(alphaarr[bob])) clf = SGDClassifier(loss="log", penalty="l1", alpha=0.08) ress = clf.fit(X, y) #print('clf predict',clf.predict(X)) #print('score---',clf.score(X,y)) #print('sparcify---',clf.sparsify()) #model = fs.SelectFromModel(SGDClassifier(loss="log", penalty="l1", alpha=float(alphaarr[bob])).fit(X, y),prefit=True) model = fs.SelectFromModel(ress, prefit=True) X_new = model.transform(X) print('orgin --', X.shape) print('new --', X_new.shape) ret = clf.predict(X) indx_ret1 = [index for index, value in enumerate(ret) if value == 1] indx_y1 = [index for index, value in enumerate(y) if value == 1] indx_ret0 = [index for index, value in enumerate(ret) if value == 0] indx_y0 = [index for index, value in enumerate(y) if value == 0] print( 'len total', len(list(set(indx_ret0) & set(indx_y0))) + len(list(set(indx_ret1) & set(indx_y1)))) return {"bestFeatureSummary": indx_ret1}
# Creating the regression model from sklearn.model_selection import train_test_split from sklearn.linear_model import LinearRegression from sklearn.linear_model import Ridge from sklearn.linear_model import Lasso from sklearn.neighbors import KNeighborsRegressor from sklearn.svm import LinearSVR from sklearn.neural_network import MLPRegressor from sklearn.metrics import r2_score from sklearn.metrics import mean_squared_error regressors = { 'LINEAR REGRESSION': LinearRegression(), 'RIDGE REGRESSION': Ridge(alpha=100), 'LASSO REGRESSION': Lasso(alpha=0.001), 'K-NEIGHBORS REGRESSION': KNeighborsRegressor(n_neighbors=8), 'SVR REGRESSION': LinearSVR(C=0.01, max_iter=10000000), 'MLP REGRESSION': MLPRegressor(max_iter=6000) } def regression_analysis(): # split data into training and testing X_train, X_test, y_train, y_test = train_test_split(features, target, random_state=3000) for regressor_item, regressor_object in regressors.items():
def Lasso_regression(degree, alpha): return Pipeline([("poly", PolynomialFeatures(degree=degree)), ("stand", StandardScaler()), ("lasso", Lasso(alpha=alpha))])
acc_gbr = [] for i in range(0, len(y_pred_gbr)): acc_gbr.append(abs(y_pred_gbr[i] - Y_test[i]) / Y_test[i]) final_s_gbr = sum(acc_gbr) / len(acc_gbr) acc_train_gbr = [] for i in range(0, len(y_pred_train_gbr)): acc_train_gbr.append(abs(y_pred_train_gbr[i] - Y_train[i]) / Y_train[i]) final_s_train_gbr = sum(acc_train_gbr) / len(acc_train_gbr) final_acc_gbr = (1 - final_s_train_gbr) * 100 print("Accuracy of GradientBoostRegression is") print(final_acc_gbr) print("The mean absolute error of GradientBoost ") mae_gbr = mean_absolute_error(Y_test, y_pred_gbr) print(mae_gbr) model = Lasso() visualizer1 = PredictionError(modelgb) visualizer1.fit(X_train, Y_train) # Fit the training data to the visualizer visualizer1.score(X_test, Y_test) # Evaluate the model on the test data g = visualizer1.poof() from sklearn.ensemble import RandomForestRegressor rfregressor = RandomForestRegressor(n_estimators=100, random_state=0) modelrfr = rfregressor.fit(X_train, Y_train) y_pred_rfr = rfregressor.predict(X_test) y_pred_train_rfr = rfregressor.predict(X_train) y_pred_train_rfr = y_pred_train_rfr.tolist() acc_rfr = [] for i in range(0, len(y_pred_rfr)): acc_rfr.append(abs(y_pred_rfr[i] - Y_test[i]) / Y_test[i])
name = 'BayesianRidge' clf = BayesianRidge() clf.fit(X_train, y_train) models[name] = clf # Ridge for a in [1e-3, 1e-2, 1e-1, 1, 10, 100]: name = f'Ridge-alph-{a:.0E}' clf = Ridge(alpha=a) clf.fit(X_train, y_train) models[name] = clf # Lasso for a in [1e-3, 1e-2, 1e-1, 1]: name = f'Lasso-alph-{a:.0E}' clf = Lasso(alpha=a) clf.fit(X_train, y_train) models[name] = clf # SGDRegressor name = 'SGDRegressor' clf = SGDRegressor() clf.fit(X_train, y_train) models[name] = clf # Quadratic Regression for n in [2, 3]: name = f'QuadraticRegr-{n}' clf = make_pipeline(PolynomialFeatures(n), Ridge()) clf.fit(X_train, y_train) models[name] = clf
# -------------- # import libraries from sklearn.model_selection import GridSearchCV, RandomizedSearchCV from sklearn.linear_model import Ridge, Lasso # regularization parameters for grid search ridge_lambdas = [0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6, 10, 30, 60] lasso_lambdas = [ 0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1 ] # Code starts here ridge_model = Ridge() lasso_model = Lasso() ridge_grid = GridSearchCV(estimator=ridge_model, param_grid=dict(alpha=ridge_lambdas)) lasso_grid = GridSearchCV(estimator=lasso_model, param_grid=dict(alpha=lasso_lambdas)) ridge_grid.fit(X_train, y_train) lasso_grid.fit(X_train, y_train) ridge_grid_pred = ridge_grid.predict(X_test) lasso_grid_pred = lasso_grid.predict(X_test) ridge_rmse = np.sqrt(mean_squared_error(y_test, ridge_grid_pred)) lasso_rmse = np.sqrt(mean_squared_error(y_test, lasso_grid_pred))