def test_features_in_secondary(): lr = LinearRegression() svr_lin = SVR(kernel='linear') rf = RandomForestRegressor(random_state=2) ridge = Ridge(random_state=0) svr_rbf = SVR(kernel='rbf') stack = StackingRegressor(regressors=[svr_lin, lr, ridge, rf], meta_regressor=svr_rbf, use_features_in_secondary=True) stack.fit(X1, y).predict(X1) mse = 0.14 got = np.mean((stack.predict(X1) - y)**2) print(got) assert round(got, 2) == mse stack = StackingRegressor(regressors=[svr_lin, lr, ridge, rf], meta_regressor=svr_rbf, use_features_in_secondary=False) # dense stack.fit(X1, y).predict(X1) mse = 0.12 got = np.mean((stack.predict(X1) - y)**2) print(got) assert round(got, 2) == mse
def test_features_in_secondary(): lr = LinearRegression() svr_lin = SVR(kernel='linear', gamma='auto') rf = RandomForestRegressor(n_estimators=10, random_state=2) ridge = Ridge(random_state=0) svr_rbf = SVR(kernel='rbf', gamma='auto') stack = StackingRegressor(regressors=[svr_lin, lr, ridge, rf], meta_regressor=svr_rbf, use_features_in_secondary=True) stack.fit(X1, y).predict(X1) mse = 0.14 got = np.mean((stack.predict(X1) - y) ** 2) print(got) assert round(got, 2) == mse stack = StackingRegressor(regressors=[svr_lin, lr, ridge, rf], meta_regressor=svr_rbf, use_features_in_secondary=False) # dense stack.fit(X1, y).predict(X1) mse = 0.12 got = np.mean((stack.predict(X1) - y) ** 2) print(got) assert round(got, 2) == mse
def test(self): df =pd.read_csv('MorganMACCS.csv') baseDf = df extractDf = df['CAS'].isin(ejectCAS) df = df[~df['CAS'].isin(ejectCAS)] y = df['logTox'] dropList = ['CAS','toxValue','logTox','HDonor', 'HAcceptors', 'AromaticHeterocycles', 'AromaticCarbocycles', 'FractionCSP3'] #dropList = ['CAS','toxValue','logTox'] X = df.drop(columns=dropList) #Normalize for name in X.columns: if str.isdecimal(name)==True: if X[str(name)].sum() == 0: print(name) X = X.drop(columns=name) else: std =X[name].std() mean = X[name].mean() X[name] = X[name].apply(lambda x: ((x - mean) * 1 / std + 0)) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.1, random_state=2) cols = np.arange(1,550,1).tolist() cols = X.columns.tolist() cols = [1,2,3] # Initializing Classifiers reg1 = Ridge(random_state=1) #reg2 = ExtraTreesRegressor() reg2 = ExtraTreesRegressor(n_estimators=50,max_features= 50,min_samples_split= 5,max_depth= 50, min_samples_leaf= 5) reg3 = SVR(gamma='auto',kernel='linear') reg4 = LGBMRegressor(boosting_type='gbdt', num_leaves= 60,learning_rate=0.06) pls = PLSRegression(n_components=3) pipe1 = make_pipeline(ColumnSelector(cols=cols), ExtraTreesRegressor(n_estimators=50)) #linear =SGDRegressor(max_iter=1000) rgf = RGFRegressor(max_leaf=1000, algorithm="RGF",test_interval=100, loss="LS",verbose=False,l2=1.0) nbrs = KNeighborsRegressor(2) pipe2 = make_pipeline(ColumnSelector(cols=cols), KNeighborsRegressor(31)) meta = ExtraTreesRegressor(n_estimators=50,max_features= 7,min_samples_split= 5,max_depth= 50, min_samples_leaf= 5) stackReg = StackingRegressor(regressors=[reg1,reg2, reg3,pipe1,pls,nbrs,rgf], meta_regressor=meta,verbose=1) stackReg.fit(X_train, y_train) y_pred = stackReg.predict(X_train) y_val = stackReg.predict(X_test) print("Root Mean Squared Error train: %.4f" % calcRMSE(y_pred,y_train)) print("Root Mean Squared Error test: %.4f" % calcRMSE(y_val,y_test)) print('Correlation Coefficient train: %.4f' % calcCorr(y_pred,y_train)) print('Correlation Coefficient test: %.4f' % calcCorr(y_val,y_test)) reg4.fit(X_train, y_train) y_pred = reg4.predict(X_train) y_val = reg4.predict(X_test) print("Root Mean Squared Error train: %.4f" % calcRMSE(y_pred,y_train)) print("Root Mean Squared Error test: %.4f" % calcRMSE(y_val,y_test)) print('Correlation Coefficient train: %.4f' % calcCorr(y_pred,y_train)) print('Correlation Coefficient test: %.4f' % calcCorr(y_val,y_test))
class RegressorBlender: def __init__(self, x_train, x_test, y_train, y_test=None): x_train.drop(['Unnamed: 0', 'Id'], axis=1, inplace=True) x_test.drop(['Unnamed: 0', 'Id'], axis=1, inplace=True) self.x_train = x_train self.x_test = x_test self.y_train = y_train['y'].values if self.y_train is not None: self.y_test = y_test['y'].values def reg_blend(self): mete_reg = LinearRegression() reg1 = model.svm_regressor() reg2 = model.randomforest_regressor() reg3 = model.xgb_regressor() self.blend = StackingRegressor(regressors=[reg1, reg2, reg3], meta_regressor=mete_reg) self.blend.fit(self.x_train, self.y_train) return self.blend def score(self): scores = cross_val_score(self.blend, X=self.x_train, y=self.y_train, cv=10, verbose=2) return scores def prediction(self): y_pred = self.blend.predict(self.x_test) return y_pred
class Blend: def __init__(self, x_train, x_test, y_train): x_train.drop(['Unnamed: 0', 'PromoInterval', 'Date'], axis=1, inplace=True) x_test.drop(['Unnamed: 0', 'Id', 'PromoInterval', 'Date'], axis=1, inplace=True) self.x_train = x_train self.x_test = x_test self.y_train = y_train['Sales'].values def blending(self): mete_reg = LinearRegression() reg1 = model.svm_regressor() reg2 = model.randomforest_regressor() reg3 = model.xgb_regressor() self.blend = StackingRegressor(regressors=[reg1, reg2, reg3], meta_regressor=mete_reg) self.blend.fit(self.x_train, self.y_train) return self.blend def score(self): scores = cross_val_score(self.blend, X=self.x_train, y=self.y_train, cv=10, verbose=2) # scoring='neg_mean_squared_error' return scores def prediction(self): y_pred = self.blend.predict(self.x_test) y_pred = np.expm1(y_pred) return y_pred
def test_predict_meta_features(): lr = LinearRegression() svr_rbf = SVR(kernel='rbf') ridge = Ridge(random_state=1) stregr = StackingRegressor(regressors=[lr, ridge], meta_regressor=svr_rbf) X_train, X_test, y_train, y_test = train_test_split(X2, y, test_size=0.3) stregr.fit(X_train, y_train) test_meta_features = stregr.predict(X_test) assert test_meta_features.shape[0] == X_test.shape[0]
def test_multivariate_class(): lr = LinearRegression() ridge = Ridge(random_state=1) meta = LinearRegression(normalize=True) stregr = StackingRegressor(regressors=[lr, ridge], meta_regressor=meta) stregr.fit(X2, y2).predict(X2) mse = 0.122 got = np.mean((stregr.predict(X2) - y2)**2) assert round(got, 3) == mse
def stacking(self): from sklearn.svm import SVR from sklearn.pipeline import make_pipeline from sklearn.preprocessing import RobustScaler, MinMaxScaler from sklearn.preprocessing import StandardScaler from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor from xgboost import XGBRegressor import lightgbm as lgb from lightgbm import LGBMRegressor import xgboost as xgb from mlxtend.regressor import StackingRegressor import scipy as sc s = make_pipeline(RobustScaler(), SVR(kernel='rbf', C=10, gamma=0.005)) rf = make_pipeline( RandomForestRegressor(random_state=641, n_estimators=250, max_depth=9)) GBoost = GradientBoostingRegressor(n_estimators=330, learning_rate=0.01, max_depth=12, max_features='sqrt', min_samples_leaf=1, min_samples_split=42, loss='ls', random_state=40, subsample=1) model_xgb = xgb.XGBRegressor(colsample_bytree=1, gamma=5, learning_rate=0.01, max_depth=11, min_child_weight=1.7817, n_estimators=500, reg_alpha=0.8, reg_lambda=5, subsample=0.5213, silent=1, seed=1024, nthread=-1) model_lgb = LGBMRegressor(objective='regression', num_leaves=4, learning_rate=0.05, n_estimators=290, max_bin=147, subsample=0.65, colsample_bytree=0.7, feature_fraction_seed=46, subsample_freq=9, min_child_samples=20, min_child_weight=0.001) regressors = [s, rf, GBoost, model_lgb, model_xgb] stregr = StackingRegressor(regressors=regressors, meta_regressor=model_xgb) stregr.fit(self.X_train, self.y_train) print("the model is staking and the test's pearsonr is: ", sc.stats.pearsonr(self.y_test, stregr.predict(self.X_test))[0]) return stregr
def test_predict_meta_features(): lr = LinearRegression() svr_rbf = SVR(kernel='rbf') ridge = Ridge(random_state=1) stregr = StackingRegressor(regressors=[lr, ridge], meta_regressor=svr_rbf) X_train, X_test, y_train, y_test = train_test_split(X2, y, test_size=0.3) stregr.fit(X_train, y_train) test_meta_features = stregr.predict(X_test) assert test_meta_features.shape[0] == X_test.shape[0]
def test_multivariate_class(): lr = LinearRegression() ridge = Ridge(random_state=1) meta = LinearRegression(normalize=True) stregr = StackingRegressor(regressors=[lr, ridge], meta_regressor=meta) stregr.fit(X2, y2).predict(X2) mse = 0.122 got = np.mean((stregr.predict(X2) - y2) ** 2) assert round(got, 3) == mse
def test_multivariate(): lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) stregr.fit(X2, y).predict(X2) mse = 0.218 got = np.mean((stregr.predict(X2) - y) ** 2) assert round(got, 3) == mse
def test_multivariate(): lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) stregr.fit(X2, y).predict(X2) mse = 0.22 got = np.mean((stregr.predict(X2) - y)**2) assert round(got, 2) == mse
def test_different_models(): lr = LinearRegression() svr_lin = SVR(kernel='linear', gamma='auto') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf', gamma='auto') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) stregr.fit(X1, y).predict(X1) mse = 0.21 got = np.mean((stregr.predict(X1) - y)**2) assert round(got, 2) == mse
def test_multivariate_class(): lr = LinearRegression() ridge = Ridge(random_state=1) meta = LinearRegression(normalize=True) stregr = StackingRegressor(regressors=[lr, ridge], meta_regressor=meta) stregr.fit(X2, y2).predict(X2) mse = 0.12 got = np.mean((stregr.predict(X2) - y2)**2.) # there seems to be an issue with the following test on Windows # sometimes via Appveyor assert round(got, 2) == mse, got
def test_different_models(): lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) y_pred = stregr.fit(X1, y).predict(X1) mse = 0.214 got = np.mean((stregr.predict(X1) - y) ** 2) assert round(got, 3) == mse
def test_multivariate_class(): lr = LinearRegression() ridge = Ridge(random_state=1) meta = LinearRegression(normalize=True) stregr = StackingRegressor(regressors=[lr, ridge], meta_regressor=meta) stregr.fit(X2, y2).predict(X2) mse = 0.12 got = np.mean((stregr.predict(X2) - y2) ** 2.) # there seems to be an issue with the following test on Windows # sometimes via Appveyor assert round(got, 2) == mse, got
def mlx_reg_1(self): lr, lr_pred = self.linear_regr() rf, rf_pred = self.random_forest_regr() lasso, lasso_pred = self.lasso_regr() sclf = StackingRegresorMLX( regressors=[lr, rf, lasso], meta_regressor=RandomForestRegressor(ccp_alpha=0.1, max_features="auto", n_estimators=30) ) sclf.fit(self.x_train, self.y_train) return sclf.predict(self.x_test)
def test_sparse_matrix_inputs_and_features_in_secondary(): lr = LinearRegression() svr_lin = SVR(kernel='linear', gamma='auto') rf = RandomForestRegressor(n_estimators=10, random_state=2) ridge = Ridge(random_state=0) svr_rbf = SVR(kernel='rbf', gamma='auto') stack = StackingRegressor(regressors=[svr_lin, lr, ridge, rf], meta_regressor=svr_rbf, use_features_in_secondary=True) # dense stack.fit(X1, y).predict(X1) mse = 0.14 got = np.mean((stack.predict(X1) - y)**2) assert round(got, 2) == mse # sparse stack.fit(sparse.csr_matrix(X1), y) mse = 0.14 got = np.mean((stack.predict(sparse.csr_matrix(X1)) - y)**2) assert round(got, 2) == mse
def regressionStacking(df): # StackingRegressor inputdata type is ndarray X_train, X_test, y_train, y_test = trainDataSplit(df) randomforest_regressor = RandomForestRegressor() # # lightgbm不是scikit-learn的包,mlxtend不支持 # lgb_train = lightgbm.Dataset(X_train, y_train) # lgb_eval = lightgbm.Dataset(X_test, y_test, reference=lgb_train) # # # specify your configurations as a dict # params = { # 'task': 'train', # 'boosting_type': 'gbdt', # 'objective': 'regression', # 'metric': {'l2', 'auc'}, # 'num_leaves': 2 ** 10, # 'learning_rate': 1.0, # 'feature_fraction': 0.9, # 'bagging_fraction': 0.8, # 'bagging_freq': 5, # 'verbose': 0 # } # lightgbm_regressor = lightgbm.train(params, # lgb_train, # num_boost_round=20, # valid_sets=lgb_eval, # early_stopping_rounds=5) lasso_regressor = Lasso() dnn_regressor = MLPRegressor() linearRegression_regressor = LinearRegression() stacking_regressor = StackingRegressor( regressors=[randomforest_regressor, lasso_regressor, dnn_regressor], meta_regressor=linearRegression_regressor) stacking_regressor.fit(X_train, X_train) y_pred = stacking_regressor.predict(X_test) criterion_df, predict_result = predictResultOutput(stacking_regressor, X_test, y_test, y_pred) # save model joblib.dump(stacking_regressor, 'stacking.model') return criterion_df, predict_result
def sbg_mlxtend_ensamble(iterate): iterate += 501 lin_mod = linear_model.LinearRegression() bsn_rdg = linear_model.BayesianRidge() elstc_nt = ElasticNet(alpha=0.2, l1_ratio=1) ridge = Ridge(alpha=0.01, tol=0.1, solver='sag') svr_rbf = svm.SVR(kernel='rbf', C=1e3, gamma=0.1) sgd_reg = linear_model.SGDRegressor(penalty='l2', alpha=0.001, n_iter=1000) lasso_reg = linear_model.Lasso(alpha=1, max_iter=3000, normalize='True', selection='random', tol=0.001) rndm_frst = RandomForestRegressor(max_depth=5, n_estimators=10) stregr = StackingRegressor(regressors=[sgd_reg, rndm_frst], meta_regressor=ridge) X_train, X_test, y_train, y_test = train_test_split(df_X, df_Y2, test_size=0.20, random_state=iterate) X_train = sc.fit_transform(X_train) X_test = sc.transform(X_test) stregr.fit(X_train, y_train) y_pred = stregr.predict(X_test) #print("Mean Squared Error: %.4f" # % np.mean((y_pred - y_test.values) ** 2)) #print('Variance Score: %.4f' % stregr.score(X_test, y_test.values)) dev_Memory = abs(y_pred - y_test.values) mean_dev = np.mean(dev_Memory) mse_Memory = np.sqrt(np.sum(dev_Memory**2) / dev_Memory.size) mape = np.mean(dev_Memory / y_test.values) max_pe = np.max(dev_Memory) max_ne = np.max(np.negative(dev_Memory)) new_data1 = pd.DataFrame(y_pred) new_data2 = pd.DataFrame(y_test.values) new_data = pd.merge(new_data1, new_data2, left_index=True, right_index=True) filename12 = r'C:\Users\epatdeb\AlphaCANDI\SBG_Rawinput_1.6\latest\Logs\AlphaCandi17_MlxEnsmbl_Memory.log' logging.basicConfig(filename=filename12, level=logging.DEBUG) logging.info( "tensor_bp sbg_mlxtend_ensamble iter:%s \n \n y_pred/y_test: \n %s \n mae:%s mse:%s mape:%s max_pe:%s max_ne:%s", iterate, new_data, mean_dev, mse_Memory, mape, max_pe, max_ne) logging.shutdown() return mean_squared_error(y_test, y_pred), mean_dev, mape
def Gbc(): from sklearn.ensemble import GradientBoostingClassifier, AdaBoostRegressor from sklearn.linear_model import LogisticRegression from mlxtend.regressor import StackingRegressor from sklearn.svm import SVR adaboost = AdaBoostRegressor() lr = LogisticRegression gb = GradientBoostingClassifier() svr = SVR(kernel='linear') svr_rbf = SVR(kernel='rbf') regressors = [svr, adaboost, gb] stregr = StackingRegressor(regressors=regressors, meta_regressor=svr_rbf) stregr.fit(X_train, y_train) outpred = stregr.predict(X_valid) evaluate_strategy(outpred)
def test_sample_weight(): lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) pred1 = stregr.fit(X1, y, sample_weight=w).predict(X1) mse = 0.22 got = np.mean((stregr.predict(X1) - y)**2) assert round(got, 2) == mse # make sure that this is not equivalent to the model with no weight pred2 = stregr.fit(X1, y).predict(X1) maxdiff = np.max(np.abs(pred1 - pred2)) assert maxdiff > 1e-3, "max diff is %.4f" % maxdiff
def test_sample_weight(): lr = LinearRegression() svr_lin = SVR(kernel='linear', gamma='auto') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf', gamma='auto') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) pred1 = stregr.fit(X1, y, sample_weight=w).predict(X1) mse = 0.22 got = np.mean((stregr.predict(X1) - y) ** 2) assert round(got, 2) == mse # make sure that this is not equivalent to the model with no weight pred2 = stregr.fit(X1, y).predict(X1) maxdiff = np.max(np.abs(pred1 - pred2)) assert maxdiff > 1e-3, "max diff is %.4f" % maxdiff
def train_model(X_train, y_train): clf1 = LinearSVR() clf2 = LinearRegression() clf3 = Ridge() clf4 = LGBMRegressor() svr_linear = LinearSVR() sr = StackingRegressor(regressors=[clf1, clf2, clf3, clf4], meta_regressor=svr_linear) sr.fit(X_train, y_train) result = sr.predict(X_train) score = get_rmse_score(result, y_train) print("RMSE Score train: %.4f" % score) return sr
def train(self, X, y): features = X labels = y #test train split X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.25, random_state=4) #Ridge regcv = linear_model.RidgeCV( alphas=[0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]) regcv.fit(features, labels) regcv.alpha_ reg = linear_model.Ridge(alpha=regcv.alpha_) reg.fit(features, labels) # GB params = { 'n_estimators': 100, 'max_depth': 5, 'min_samples_split': 2, 'learning_rate': 0.1, 'loss': 'ls' } gbr = ensemble.GradientBoostingRegressor(**params) gbr.fit(features, labels) #blended model meta = linear_model.LinearRegression() blender = StackingRegressor(regressors=[reg, gbr], meta_regressor=meta) _ = blender.fit(features, labels) y_pred = blender.predict(X_test) print "***** TRAINING STATS ********" scores = cross_val_score(blender, features, labels, cv=10) print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2)) mean_diff = np.mean(np.abs(np.exp(Y_test) - np.exp(y_pred))) p_mean_diff = np.mean(mean_diff / np.exp(Y_test)) print "Mean Error:\t %.0f/%0.3f%%" % (mean_diff, p_mean_diff * 100) print "***** TRAINING STATS ********" return blender
def train(self, X,y): features = X labels = y #test train split X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.25, random_state=4) #Ridge regcv = linear_model.RidgeCV(alphas=[0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]) regcv.fit(features, labels) regcv.alpha_ reg = linear_model.Ridge(alpha=regcv.alpha_) reg.fit(features, labels) # GB params = {'n_estimators': 100, 'max_depth': 5, 'min_samples_split': 2, 'learning_rate': 0.1, 'loss': 'ls'} gbr = ensemble.GradientBoostingRegressor(**params) gbr.fit(features, labels) #blended model meta = linear_model.LinearRegression() blender = StackingRegressor(regressors=[reg, gbr], meta_regressor=meta) _=blender.fit(features, labels) y_pred = blender.predict(X_test) print "***** TRAINING STATS ********" scores = cross_val_score(blender, features, labels, cv=10) print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2)) mean_diff = np.mean(np.abs(np.exp(Y_test)-np.exp(y_pred))) p_mean_diff = np.mean(mean_diff/np.exp(Y_test)) print "Mean Error:\t %.0f/%0.3f%%" % (mean_diff, p_mean_diff*100) print "***** TRAINING STATS ********" return blender
def stackModel(self): train_X = self.X.as_matrix() train_Y = self.Y.as_matrix() test_X = self.Test.as_matrix() # train_X = data_scaler(train_X) X_train, X_test, y_train, y_test = train_test_split(train_X, train_Y, test_size=0.2, random_state=1) gbdt = GradientBoostingRegressor(loss='ls', alpha=0.9, n_estimators=500, learning_rate=0.05, max_depth=8, subsample=0.8, min_samples_split=9, max_leaf_nodes=10) xgb = XGBRegressor(max_depth=5, n_estimators=500, learning_rate=0.05, silent=False) lr = LinearRegression() rfg = RandomForestRegressor(bootstrap=False, max_features=0.05, min_samples_leaf=11, min_samples_split=8, n_estimators=100) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[gbdt, xgb, lr, rfg], meta_regressor=svr_rbf) stregr.fit(X_train, y_train) stregr.predict(X_train) # Evaluate and visualize the fit print("Mean Squared Error: %.6f" % np.mean((stregr.predict(X_train) - y_train) ** 2) ** 0.5) error(stregr.predict(X_test), y_test) # online result = stregr.predict(test_X) save_to_file(result, self.uid, "../result/result_12.09_2_stacking.csv") with plt.style.context(('seaborn-whitegrid')): plt.scatter(X_train, y_train, c='lightgray') plt.plot(X_train, stregr.predict(X_train), c='darkgreen', lw=2) plt.show()
plt.ylabel('Accuracy') plt.show() # In[368]: from mlxtend.regressor import StackingRegressor lr = LinearRegression() sclf = StackingRegressor(regressors=[grid_search, abr, rfr], meta_regressor=lr) print('3-fold cross validation:\n') for clf, label in zip([grid_search, abr, rfr, sclf], ['grid_search', 'abr', 'rfr', 'StackingClassifier']): scores = cross_val_score(clf, X, y) print("Accuracy: %0.2f (+/- %0.2f) [%s]" % (scores.mean(), scores.std(), label)) sclf.fit(X_train, y_train) predictions = sclf.predict(X_test) # In[370]: train_sizes, train_score, test_score = learning_curve( sclf, X, y, train_sizes=[0.1, 0.2, 0.4, 0.6, 0.8, 1], cv=3) train_error = 1 - np.mean(train_score, axis=1) test_error = 1 - np.mean(test_score, axis=1) plt.plot(train_sizes, 1 - train_error, 'o-', color='r', label='training') plt.plot(train_sizes, 1 - test_error, 'o-', color='g', label='testing') plt.legend(loc='best') plt.xlabel('traing examples') plt.ylabel('Accuracy') plt.show()
num_boost_round=2889, early_stopping_rounds=50, evals=watchlist) rfreg = RandomForestRegressor(random_state=1, max_depth=15) ridge_reg = Ridge(normalize=True) lasso_reg = Lasso() linear_reg = LinearRegression(normalize=True) stacking_reg = StackingRegressor(regressors=[rfreg, ridge_reg, lasso_reg], meta_regressor=linear_reg) feature = [x for x in train_zero_var.columns if x not in ['Value']] # X_train, X_test, y_train, y_test = train_test_split(train_zero_var[feature], train_zero_var['Value'], test_size=0.2, # random_state=0) stacking_reg.fit(X_train, y_train) stacking_test = pd.DataFrame(stacking_reg.predict(X_test)) stacking_test.columns = ['stacking_pred'] y_test = pd.DataFrame(y_test) y_test.columns = ['Value'] mean_squared_error(stacking_test['stacking_pred'], y_test['Value']) train_zero_var = train_zero_var.reset_index() # predict for Random Forest rf_pred = pd.DataFrame() for idx in range(0, 5): train = train_zero_var[train_zero_var['index'] % 5 != idx] test = train_zero_var[train_zero_var['index'] % 5 == idx] stacking_feature = [ x for x in train.columns if x not in ['index', 'Value'] ]
nthread=-1) gbm_b = GradientBoostingRegressor(learning_rate=0.05, n_estimators=2000, max_depth=4, max_features='log2', min_samples_leaf=15, min_samples_split=10, loss='huber') stackmodel = StackingRegressor( regressors=[ElNet_b, lasso_b, ridge_b, svr_b, model_xgb_b, gbm_b], meta_regressor=Lasso(alpha=0.00035)) stackmodel.fit(x_train, y_train) stacked = stackmodel.predict(x_test) rmse_stacked = np.sqrt(mean_squared_error(y_train, stackmodel.predict(x_train))) stacked_pred = np.expm1(stacked) # Averaged model ensembled = np.expm1((0.25 * ridge.predict(x_test).reshape(-1, 1)) + (0.2 * ElNet.predict(x_test).reshape(-1, 1)) + (0.2 * lasso.predict(x_test).reshape(-1, 1)) + (0.15 * model_xgb.predict(x_test).reshape(-1, 1)) + (0.2 * GBoost.predict(x_test).reshape(-1, 1))) # Print the performance of each model obj = pd.DataFrame([[ score_ridge, rdg_trainRMSE, rdg_testRMSE, rmse_ridge_test - rmse_ridge_train, 0.11866
def predict(): ''' For rendering results on HTML GUI ''' features = [x for x in request.form.values()] #final_features = [np.array(int_features)] #prediction = model.predict(final_features) #output = round(prediction[0], 2) features = np.array(features) features = features.reshape(1, 6) features = pd.DataFrame(data=features, columns=[ 'Name', 'Genre', 'Comments', 'Likes', 'Popularity', 'Followers' ]) df = pd.read_csv('data.csv') cv = {'Comments': int, 'Likes': int, 'Popularity': int, 'Followers': int} df = df.astype(cv) features = features.astype(cv) #x=df[df['Views']==0].index df.drop(index=df[df['Views'] < df['Likes']].index, axis=1, inplace=True) df.drop(index=df[df['Views'] < df['Comments']].index, axis=1, inplace=True) df.drop(index=df[df['Views'] < df['Popularity']].index, axis=1, inplace=True) Q1 = df.quantile(0.25) Q3 = df.quantile(0.75) IQR = Q3 - Q1 (df < (Q1 - 1.5 * IQR)) | (df > (Q3 + 1.5 * IQR)) df = df[~((df < (Q1 - 3 * IQR)) | (df > (Q3 + 3 * IQR))).any(axis=1)] df = df.drop( columns=['Unique_ID', 'Country', 'Song_Name', 'Timestamp', 'index']) y = df['Views'] df = df.drop(columns=['Views']) be = BinaryEncoder() df = be.fit_transform(df) f = be.transform(features) X = df.iloc[:, :] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0) rg1 = AdaBoostRegressor() rg1.fit(X_train, y_train) #ypred=rg1.predict(X_test) #sqrt(mean_squared_error(y_test,ypred)) rg2 = GradientBoostingRegressor(n_estimators=300, learning_rate=0.1) # para={'n_estimators':[250,300],'learning_rate':[1,0.1,0.01]} # grid=GridSearchCV(estimator=rg8,param_grid=para,verbose=1,cv=10,n_jobs=-1) rg2.fit(X_train, y_train) #ypred=rg2.predict(X_test) #sqrt(mean_squared_error(y_test,ypred)) rg3 = RandomForestRegressor(random_state=0, n_estimators=20, max_depth=15) # para={'n_estimators':[5,10,30,20],'max_depth':[5,8,20,17]} # grid=GridSearchCV(estimator=rg9,param_grid=para,cv=10,verbose=1,n_jobs=-1) rg3.fit(X_train, y_train) #ypred=rg3.predict(X_test) #sqrt(mean_squared_error(y_test,ypred)) rg6 = StackingRegressor([rg1, rg2], meta_regressor=rg3) rg6.fit(X_train, y_train) #ypred=rg6.predict(X_test) #sqrt(mean_squared_error(y_test,ypred)) f = f.iloc[:, :] y_pred = rg6.predict(f) y_pred = y_pred.astype(int) return render_template( 'index.html', prediction_text='Numberb of Views is {}'.format(y_pred))
#============================================================================== # 4) LGBMRegressor模型 #============================================================================== # from lightgbm import LGBMRegressor # # model_lgb = LGBMRegressor() #============================================================================== # 5) 融合模型 #============================================================================== from mlxtend.regressor import StackingRegressor # regressors = [model_xgb,model_rfg,model_gb] model = StackingRegressor(regressors=regressors, meta_regressor=model_xgb) # model = model_gb model.fit(train_text,train_labels) # print('The parameters of the best model are: ') # print(model.best_params_) preds = model.predict(train_text) print('The pearsonr of training set is {}'.format(pearsonr (list(train_labels), list(preds))[0])) print('The MSE of training set is {}'.format(mean_squared_error(list(train_labels), list(preds)))) #============================================================================== # 预测 测试集 #============================================================================== preds = model.predict(test_text) print('The pearsonr of test set is {}'.format(pearsonr (list(test_labels), list(preds))[0])) print('The MSE of test set is {}'.format(mean_squared_error(list(test_labels), list(preds))))
'max_bin': 8192, 'verbosity': 10 } modelL1 = lgb.LGBMRegressor(**params) modelL2 = lgb.LGBMRegressor(**params2) metaregr = Ridge(solver="sag", max_iter=300) stregr = StackingRegressor(regressors=[modelR1, modelR3, modelL1, modelL2], meta_regressor=metaregr, verbose=10) stregr.fit(X, y) print('Weights/Iter of Regressors=', stregr.coef_) #preds = stregr.predict(X) #print('RMSE=', mean_squared_error(y, preds)**0.5) pred_test = stregr.predict(X_test) submission['price'] = np.expm1(pred_test) submission.to_csv("FF_LR_Meta.csv", index=False) #============================================================================== # GridSearch Cross Validation # paramsGSCV = { # 'ridge__alpha': [0.4, 1.0], # 'lgbmregressor__max_depth': [4, 5], # 'meta-ridge__alpha': [1.0] # } # stregr.get_params().keys() #Print list of available parameters # grid = GridSearchCV(estimator=stregr, # param_grid=paramsGSCV, # cv=4,
y[::5] += 3 * (0.5 - np.random.rand(8)) # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) # Training the stacking classifier stregr.fit(X, y) stregr.predict(X) # Evaluate and visualize the fit print("Mean Squared Error: %.4f" % np.mean((stregr.predict(X) - y)**2)) print('Variance Score: %.4f' % stregr.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, stregr.predict(X), c='darkgreen', lw=2) plt.show() # Example 2 - Stacked Regression and GridSearch from sklearn.model_selection import GridSearchCV
def main(): print("Reading in Data") #最终预测 train = pd.read_csv('cleaned_train20180129_111517.csv') test = pd.read_csv('cleaned_test20180129_111517.csv') #验证A榜结果 #train = pd.read_csv('cleaned_train20180129_102513.csv') #test = pd.read_csv('cleaned_test20180129_102513.csv') test = test.drop(['id'], axis=1) train = train.drop(['id'], axis=1) y_train = train['血糖'] #pred_proba为测试集血糖权重 threshold = 6.5 test_num = len(test) train_num = len(train) bigger_thr_X, bigger_thr_y, less_thr_X, less_thr_y, test_concat, X_train, pred_proba = fuck_columns( train, test, threshold) print("linear model 开始训练") pred_bigger, pred_less, linear_bigger, linear_less = linear_model( bigger_thr_X, bigger_thr_y, less_thr_X, less_thr_y, test_concat) #预测结果结合权重 linear_pred_res = np.array([ pred_less[i] * pred_proba[i][0] + pred_bigger[i] * pred_proba[i][1] for i in range(test_num) ]) print("lasso model 开始训练") pred_bigger, pred_less, lasso_bigger, lasso_less = lasso_model( bigger_thr_X, bigger_thr_y, less_thr_X, less_thr_y, test_concat) #预测结果结合权重 lasso_pred_res = np.array([ pred_less[i] * pred_proba[i][0] + pred_bigger[i] * pred_proba[i][1] for i in range(test_num) ]) print("ENet model 开始训练") pred_bigger, pred_less, ENet_bigger, ENet_less = ENet_model( bigger_thr_X, bigger_thr_y, less_thr_X, less_thr_y, test_concat) #预测结果结合权重 ENet_pred_res = np.array([ pred_less[i] * pred_proba[i][0] + pred_bigger[i] * pred_proba[i][1] for i in range(test_num) ]) print("集成模型开始训练...") print("RandomForestRegressor...") pred_bigger, pred_less, rf_bigger, rf_less = rf_model( bigger_thr_X, bigger_thr_y, less_thr_X, less_thr_y, test_concat) #预测结果结合权重 rf_pred_res = np.array([ pred_less[i] * pred_proba[i][0] + pred_bigger[i] * pred_proba[i][1] for i in range(test_num) ]) print("GradientBoostingRegressor...") pred_bigger, pred_less, gb_bigger, gb_less = GBoost_model( bigger_thr_X, bigger_thr_y, less_thr_X, less_thr_y, test_concat) #预测结果结合权重 gb_pred_res = np.array([ pred_less[i] * pred_proba[i][0] + pred_bigger[i] * pred_proba[i][1] for i in range(test_num) ]) print("LGBMRegressor...") pred_bigger, pred_less, lgb_bigger, lgb_less = LGBM_model( bigger_thr_X, bigger_thr_y, less_thr_X, less_thr_y, test_concat) #预测结果结合权重 lgb_pred_res = np.array([ pred_less[i] * pred_proba[i][0] + pred_bigger[i] * pred_proba[i][1] for i in range(test_num) ]) print("XGBRegressor...") pred_bigger, pred_less, xgb_bigger, xgb_less = xgb_model( bigger_thr_X, bigger_thr_y, less_thr_X, less_thr_y, test_concat) #预测结果结合权重 xgb_pred_res = np.array([ pred_less[i] * pred_proba[i][0] + pred_bigger[i] * pred_proba[i][1] for i in range(test_num) ]) ''' Stacking Learning ''' print("StackingRegressor...") stacked_averaged_bigger_models = StackingRegressor( regressors=[linear_bigger, lasso_bigger, ENet_bigger], meta_regressor=gb_bigger) stacked_averaged_less_models = StackingRegressor( regressors=[linear_less, lasso_less, ENet_less], meta_regressor=gb_less) #拟合模型 stacked_averaged_bigger_models.fit(bigger_thr_X, bigger_thr_y) stacked_averaged_less_models.fit(less_thr_X, less_thr_y) #测试集预测 stacked_bigger_pred = stacked_averaged_bigger_models.predict(test_concat) stacked_less_pred = stacked_averaged_less_models.predict(test_concat) #预测结果结合权重 stacked_pred_res = np.array([ stacked_less_pred[i] * pred_proba[i][0] + stacked_bigger_pred[i] * pred_proba[i][1] for i in range(test_num) ]) ensemble = stacked_pred_res * 0.40 + xgb_pred_res * 0.40 + lgb_pred_res * 0.20 #stacking融合linear new_ensemble = np.array([ linear_pred_res[i] * pred_proba[i][0] + ensemble[i] * pred_proba[i][1] for i in range(test_num) ]) sub = pd.DataFrame({'pred': ensemble}) sub_wig = pd.DataFrame({'pred': new_ensemble}) sub.to_csv('submission_b.csv', header=None, index=False) sub_wig.to_csv('submission_b_wig.csv', header=None, index=False)
def useXYtrain(x, y, times): flag = 0 for i in range(0, len(Selected_learnerCode)): if Selected_learnerCode[i] != '': flag += 1 if flag == 0: print('No proper learner\n') return stacking_MSE = [[], [], [], [], [], []] MSE = [[], [], [], [], [], [], []] R_square = [[], [], [], [], [], [], [], []] Ada_MSE = [] Ada_r_square = [] for i in range(0, times): print('第' + str(i + 1) + '次试验:\n') Learners_map = {} Learners = [] X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20) svr = SVR(C=1.0, epsilon=0.2) parameters = { 'C': np.logspace(-3, 3, 7), 'gamma': np.logspace(-3, 3, 7) } print("GridSearch starting...") clfsvr = GridSearchCV(svr, parameters, n_jobs=-1, scoring='neg_mean_squared_error') clfsvr.fit(X_train, y_train) print('The parameters of the best model are: ') print(clfsvr.best_params_) y_pred = clfsvr.best_estimator_.predict(X_test) # drawTrain(y_pred, y_test, 'SVR', i) # SVR_MSE.append(mean_squared_error(y_test, y_pred)) yy = clfsvr.best_estimator_.predict(x) R_square[0].append(drawTrain(y, yy, 'SVR', i)) MSE[0].append(mean_squared_error(y_test, y_pred)) if 'SVR' in Selected_learnerCode: print('SVR Mean squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") Learners.append(clfsvr.best_estimator_) Learners_map['SVR'] = svr """ann = Regressor(layers = [Layer("Sigmoid", units=14), Layer("Linear")], learning_rate = 0.02, random_state = 2018, n_iter = 10) ann.fit(X_train,y_train) y_pred = ann.predict(X_test) print('ANN Mean squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n")""" parameters = {'n_estimators': [10, 50, 100, 200, 300, 400, 500, 1000]} rfr = RandomForestRegressor(n_estimators=200, random_state=0) # drawTrain(rfr, x, y, 'RFR', i) # rfr = RandomForestRegressor(n_estimators=200, random_state=0) clfrfr = GridSearchCV(rfr, parameters, n_jobs=-1, scoring='neg_mean_squared_error') clfrfr.fit(X_train, y_train) print('The parameters of the best model are: ') print(clfrfr.best_params_) y_pred = clfrfr.best_estimator_.predict(X_test) yy = clfrfr.best_estimator_.predict(x) MSE[1].append(mean_squared_error(y_test, y_pred)) R_square[1].append(drawTrain(y, yy, 'RFR', i)) # RFR_MSE.append(mean_squared_error(y_test, y_pred)) if 'RFR' in Selected_learnerCode: print('RFR Mean squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") Learners.append(clfrfr.best_estimator_) Learners_map['RFR'] = rfr parameters = {'alpha': np.logspace(-2, 2, 5)} lasso = Lasso(alpha=0.05, random_state=1, max_iter=1000) # drawTrain(lasso, x, y, 'LASSO', i) clflasso = GridSearchCV(lasso, parameters, n_jobs=-1, scoring='neg_mean_squared_error') clflasso.fit(X_train, y_train) yy = clflasso.best_estimator_.predict(x) print('The parameters of the best model are: ') print(clflasso.best_params_) y_pred = clflasso.best_estimator_.predict(X_test) R_square[2].append(drawTrain(y, yy, 'LASSO', i)) MSE[2].append(mean_squared_error(y_test, y_pred)) if 'LASSO' in Selected_learnerCode: print('LASSO Mean squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") # file.write('LASSO Mean squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") Learners.append(clflasso.best_estimator_) Learners_map['LASSO'] = lasso # drawTrain(ENet, X_train, y_train,X_test,y_test, 'Elastic NET', i) parameters = { 'alpha': np.logspace(-2, 2, 5), 'l1_ratio': np.linspace(0, 1.0, 11) } # ENet = ElasticNet(alpha=0.05, l1_ratio=.9, random_state=3) # drawTrain(ENet, x, y, 'Elastic NET', i) ENet = ElasticNet(alpha=0.05, l1_ratio=.9, random_state=3) clfENet = GridSearchCV(ENet, parameters, n_jobs=-1, scoring='neg_mean_squared_error') clfENet.fit(X_train, y_train) print('The parameters of the best model are: ') print(clfENet.best_params_) y_pred = clfENet.best_estimator_.predict(X_test) yy = clfENet.best_estimator_.predict(x) MSE[3].append(mean_squared_error(y_test, y_pred)) R_square[3].append(drawTrain(y, yy, 'Elastic Net', i)) if 'ENET' in Selected_learnerCode: print('Elastic Net Mean squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") Learners.append(clfENet.best_estimator_) Learners_map['ENET'] = ENet parameters = {'n_estimators': [100, 500, 1000, 2000, 3000, 5000]} GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5) clfGBoost = GridSearchCV(GBoost, parameters, n_jobs=-1, scoring='neg_mean_squared_error') clfGBoost.fit(X_train, y_train) print('The parameters of the best model are: ') print(clfGBoost.best_params_) y_pred = clfGBoost.best_estimator_.predict(X_test) yy = clfGBoost.best_estimator_.predict(x) MSE[4].append(mean_squared_error(y_test, y_pred)) # GBoost_MSE.append(mean_squared_error(y_test, y_pred)) R_square[4].append(drawTrain(y, yy, 'Gradient Boosting', i)) if 'GBOOST' in Selected_learnerCode: print('GBoost squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") Learners.append(clfGBoost.best_estimator_) Learners_map['GBOOST'] = GBoost # Adaboost # Adaboost = AdaBoostRegressor(base_estimator=SVR(C=1.0, epsilon=0.2)) Adaboost = AdaBoostRegressor() Adaboost.fit(X_train, y_train) y_pred = Adaboost.predict(X_test) yy = Adaboost.predict(x) R_square[5].append(drawTrain(y, yy, 'Adaboost', i)) print('Adaboost with SVR squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") Ada_MSE.append(mean_squared_error(y_test, y_pred)) # BAGGING baggingModel = baggingAveragingModels( models=(clfsvr.best_estimator_, clfrfr.best_estimator_, clfENet.best_estimator_, clfGBoost.best_estimator_, clflasso.best_estimator_)) baggingModel.fit(X_train, y_train) y_pred = baggingModel.predict(X_test) MSE[5].append(mean_squared_error(y_test, y_pred)) yy = baggingModel.predict(x) R_square[6].append(drawTrain(y, yy, 'Bagging', i)) print('Bagging before selected squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") baggingModel = baggingAveragingModels(models=tuple(Learners)) # drawTrain(baggingModel, X_train, y_train,X_test,y_test, 'Bagging', i) # baggingModel = baggingAveragingModels(models=tuple(Learners)) baggingModel.fit(X_train, y_train) y_pred = baggingModel.predict(X_test) MSE[6].append(mean_squared_error(y_test, y_pred)) yy = baggingModel.predict(x) R_square[7].append(drawTrain(y, yy, 'Bagging', i)) print('Bagging after selected squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") stacking_R_square = [[], [], [], [], [], []] All_learner = ['SVR', 'RFR', 'LASSO', 'ENET', 'GBOOST'] for k in range(0, len(Selected_learnerCode)): """learnerList = [] for kk in range(0,len(Selected_learnerCode)): if Selected_learnerCode[kk]!='' : learnerList.append(Learners_map[Selected_learnerCode[kk]])""" """stacked_averaged_models = StackingAveragedModels(base_models=tuple(learnerList), meta_model=Learners_map[All_learner[k]]) drawTrain(stacked_averaged_models, X_train, y_train,X_test,y_test, 'stacking with '+All_learner[k], i)""" # stacked_averaged_models = StackingAveragedModels(base_models=tuple(learnerList), # meta_model=Learners_map[All_learner[k]]) params = {} """ if 'SVR' in Selected_learnerCode: params['svr__C'] = np.logspace(-3, 3, 7) params['svr__gamma'] = np.logspace(-3, 3, 7) if 'RFR' in Selected_learnerCode: params['randomforestregressor__n_estimators'] =[10, 50, 100, 500, 1000] if 'LASSO' in Selected_learnerCode: params['lasso__alpha'] = np.logspace(-2, 2, 5) if 'ENET' in Selected_learnerCode: params['elasticnet__alpha'] = np.logspace(-2, 2, 5) if 'GBOOST' in Selected_learnerCode: params['gradientboostingregressor__n_estimators']= [100, 500, 1000, 2000, 3000, 5000]""" if k == 0: params['meta-svr__C'] = np.logspace(-3, 3, 7) params['meta-svr__gamma'] = np.logspace(-3, 3, 7) if k == 1: params['meta-randomforestregressor__n_estimators'] = [ 10, 50, 100, 500, 1000 ] if k == 2: params['meta-lasso__alpha'] = np.logspace(-2, 2, 5) if k == 3: params['meta-elasticnet__alpha'] = np.logspace(-2, 2, 5) if k == 4: params['meta-gradientboostingregressor__n_estimators'] = [ 100, 500, 1000, 2000, 3000, 5000 ] """ params = {'svr__C': np.logspace(-3, 3, 7), 'svr__gamma': np.logspace(-3, 3, 7), 'randomforestregressor__n_estimators': [10, 50, 100, 500, 1000], 'lasso__alpha': np.logspace(-2, 2, 5), 'elasticnet__alpha':np.logspace(-2, 2, 5), 'gradientboostingregressor__n_estimators': [100, 500, 1000, 2000, 3000, 5000], }""" stacked_averaged_models = StackingRegressor( regressors=Learners, meta_regressor=Learners_map[All_learner[k]]) grid = GridSearchCV(estimator=stacked_averaged_models, param_grid=params) grid.fit(X_train, y_train) y_pred = grid.best_estimator_.predict(X_test) yy = grid.best_estimator_.predict(x) stacking_R_square[k].append( drawTrain(y, yy, 'stacking with ' + All_learner[k], i)) print('Stacking with metamodel is ' + All_learner[k] + ' squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") # file.write('Stacking with metamodel is lasso squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") stacking_MSE[k].append(mean_squared_error(y_test, y_pred)) # stacked_averaged_models = StackingAveragedModels(base_models=tuple(learnerList), # meta_model=baggingModel) # drawTrain(stacked_averaged_models, X_train, y_train, X_test, y_test, 'stacking with Bagging models' , i) """stacked_averaged_models = StackingAveragedModels(base_models=tuple(learnerList), meta_model=Learners_map[All_learner[k]])""" stacked_averaged_models = StackingRegressor( regressors=Learners, meta_regressor=baggingModel) # grid = GridSearchCV(estimator=stacked_averaged_models, param_grid=params) stacked_averaged_models.fit(X_train, y_train) y_pred = stacked_averaged_models.predict(X_test) yy = stacked_averaged_models.predict(x) stacking_R_square[5].append( drawTrain(y, yy, 'stacking with bagging', i)) print('Stacking with metamodel is bagging models squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") # file.write('Stacking with metamodel is lasso squared error is ' + str(mean_squared_error(y_test, y_pred)) + "\n") stacking_MSE[5].append(mean_squared_error(y_test, y_pred)) gc.collect() print("Adaboost mean is " + str(np.mean(Ada_MSE))) min_stacking_MSE = [] for i in range(0, times): minMSE = stacking_MSE[0][i] for j in range(1, 6): if stacking_MSE[j][i] < minMSE: minMSE = stacking_MSE[j][i] min_stacking_MSE.append(minMSE) plot_x = np.linspace(1, times, times) if len(MSE[0]) > 0: plt.plot(plot_x, MSE[0], 'b') if len(MSE[1]) > 0: plt.plot(plot_x, MSE[1], 'r') if len(MSE[2]) > 0: plt.plot(plot_x, MSE[2], 'y') if len(MSE[3]) > 0: plt.plot(plot_x, MSE[3], 'k') if len(MSE[4]) > 0: plt.plot(plot_x, MSE[4], 'g') if len(MSE[5]) > 0: plt.plot(plot_x, MSE[5], 'm') if len(MSE[6]) > 0: plt.plot(plot_x, MSE[6], color='coral', linestyle=':', marker='|') plt.plot(plot_x, min_stacking_MSE, color='cyan') plt.xlabel('Repeat times') plt.ylabel('MSE') plt.legend( ('SVR avg = ' + str(np.mean(MSE[0])), 'RFR avg = ' + str(np.mean(MSE[1])), 'Lasso avg=' + str(np.mean(MSE[2])), 'Enet avg=' + str(np.mean(MSE[3])), 'Gboost avg = ' + str(np.mean(MSE[4])), 'Bagging before avg = ' + str(np.mean(MSE[5])), 'Bagging after avg = ' + str(np.mean(MSE[6])), 'St-LIBS avg = ' + str(np.mean(min_stacking_MSE))), loc='upper right') plt.title('Different learning machine') plt.savefig('DifferentLearner.png') plt.clf() plt.plot() plot_x = np.linspace(1, times, times) plt.plot(plot_x, Ada_MSE, 'b') plt.plot(plot_x, MSE[6], 'r') plt.plot(plot_x, min_stacking_MSE, 'g') plt.legend(('Adaboost avg = ' + str(np.mean(Ada_MSE)), 'Bagging avg = ' + str(np.mean(MSE[6])), 'St-LIBS avg = ' + str(np.mean(min_stacking_MSE))), loc='upper right') plt.title('Bagging VS St-LIBS VS Adaboost') plt.xlabel('Repeat times') plt.ylabel('MSE') plt.savefig('Bagging VS St-LIBS&Adaboost.png') plt.clf() plt.plot() plot_x = np.linspace(1, times, times) if len(stacking_MSE[0]) > 0: plt.plot(plot_x, stacking_MSE[0], 'b') if len(stacking_MSE[1]) > 0: plt.plot(plot_x, stacking_MSE[1], 'r') if len(stacking_MSE[2]) > 0: plt.plot(plot_x, stacking_MSE[2], 'y') if len(stacking_MSE[3]) > 0: plt.plot(plot_x, stacking_MSE[3], 'k') if len(stacking_MSE[4]) > 0: plt.plot(plot_x, stacking_MSE[4], 'g') if len(stacking_MSE[5]) > 0: plt.plot(plot_x, stacking_MSE[5], 'm') plt.legend(('SVR avg = ' + str(np.mean(stacking_MSE[0])), 'RFR avg = ' + str(np.mean(stacking_MSE[1])), 'Lasso avg=' + str(np.mean(stacking_MSE[2])), 'Enet avg=' + str(np.mean(stacking_MSE[3])), 'Gboost avg = ' + str(np.mean(stacking_MSE[4])), 'Bagging avg = ' + str(np.mean(stacking_MSE[5]))), loc='upper right') plt.title('Different meta-learning machine(Adaboost avg MSE=' + str(np.mean(Ada_MSE)) + ')') plt.xlabel('Repeat times') plt.ylabel('MSE') plt.savefig('DifferentMetaLearner.png') plt.clf() plt.plot() index = ['SVR', 'RFR', 'LASSO', 'ENET', 'Gboost', 'BAGGING1', 'BAGGING2'] mse_file = pd.DataFrame(index=index, data=MSE) mse_file.to_csv('MSE.csv', encoding='utf-8') index = ['SVR', 'RFR', 'LASSO', 'ENET', 'Gboost', 'BAGGING'] mse_file = pd.DataFrame(index=index, data=stacking_MSE) mse_file.to_csv('stacking_MSE.csv', encoding='utf-8') mse_file = pd.DataFrame(data=min_stacking_MSE) mse_file.to_csv('min_stacking_MSE.csv', encoding='utf-8') index = [ 'SVR', 'RFR', 'LASSO', 'ENET', 'Gboost', 'Adaboost', 'BAGGING1', 'BAGGING2' ] r_file = pd.DataFrame(index=index, data=R_square) r_file.to_csv('R_square.csv', encoding='utf-8') index = ['SVR', 'RFR', 'LASSO', 'ENET', 'Gboost', 'BAGGING'] mse_file = pd.DataFrame(index=index, data=stacking_R_square) mse_file.to_csv('stacking_R_square.csv', encoding='utf-8')
y[::5] += 3 * (0.5 - np.random.rand(8)) # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) # Training the stacking classifier stregr.fit(X, y) stregr.predict(X) # Evaluate and visualize the fit print("Mean Squared Error: %.4f" % np.mean((stregr.predict(X) - y) ** 2)) print('Variance Score: %.4f' % stregr.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, stregr.predict(X), c='darkgreen', lw=2) plt.show() print(stregr)
model_rf = RandomForestRegressor(n_estimators=200, max_features=0.26326530612244903, criterion='mse') model_extra_tree = ExtraTreesRegressor(n_estimators=200, criterion='mse') model_gb = GradientBoostingRegressor(n_estimators=100, max_depth=5, random_state=43) model_lr = LinearRegression() svr_rbf = SVR(kernel='rbf') svr_lin = SVR(kernel='linear') ridge = Ridge() model_xgb2 = XGBRegressor(max_depth=10, n_estimators=100) model_vote = VotingClassifier( estimators=[('xgb', model_xgb), ('rf', model_rf), ('gb', model_gb)]) sclf = StackingRegressor(regressors=[model_extra_tree, model_xgb2, model_rf], meta_regressor=model_lr) time_split = TimeSeriesSplit(n_splits=5) print cross_val_score(sclf, X=train.as_matrix(), y=target.as_matrix(), scoring=SMAPE, cv=time_split).mean() sclf.fit(X=train, y=target) preds = sclf.predict(test) sample_submission['y'] = preds print sample_submission[sample_submission['y'] < 0] sample_submission['y'] = sample_submission['y'].map(lambda x: x if x > 0 else 0.0) sample_submission.to_csv("my_submission_24_2.tsv", sep=',', index=False)