Example #1
0
	def elastic(self):
		elasticNet = ElasticNetCV(l1_ratio = [0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
                          alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 
                                    0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6], 
                          max_iter = 50000, cv = 10)
		elasticNet.fit(self.X_train, self.y_train)
		alpha = elasticNet.alpha_
		ratio = elasticNet.l1_ratio_
		print("Best l1_ratio :", ratio)
		print("Best alpha :", alpha )

		print("Try again for more precision with l1_ratio centered around " + str(ratio))
		elasticNet = ElasticNetCV(l1_ratio = [ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
		                          alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6], 
		                          max_iter = 50000, cv = 10)
		elasticNet.fit(self.X_train, self.y_train)
		# l1_ratio = 1 means elastic->l1_penalty
		# l1_ratio = 0 means elastic->l2_penalty

		if (elasticNet.l1_ratio_ > 1):
		    elasticNet.l1_ratio_ = 1    
		alpha = elasticNet.alpha_
		ratio = elasticNet.l1_ratio_
		print("Best l1_ratio :", ratio)
		print("Best alpha :", alpha )

		print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio) + 
		      " and alpha centered around " + str(alpha))
		elasticNet = ElasticNetCV(l1_ratio = ratio,
		                          alphas = [alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85, alpha * .9, 
		                                    alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3, 
		                                    alpha * 1.35, alpha * 1.4], 
		                          max_iter = 50000, cv = 10)

		elasticNet.fit(self.X_train, self.y_train)
		if (elasticNet.l1_ratio_ > 1):
		    elasticNet.l1_ratio_ = 1    
		alpha = elasticNet.alpha_
		ratio = elasticNet.l1_ratio_
		print("Best l1_ratio :", ratio)
		print("Best alpha :", alpha )

		print("ElasticNet RMSE on Training set :", rmse_cv_train(elasticNet).mean())
		print("ElasticNet RMSE on Test set :", rmse_cv_test(elasticNet).mean())
Example #2
0
def elastic_net(X_train, y_train, X_test, ids):
    
    elasticNet = ElasticNetCV(l1_ratio = [0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
                              alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 
                                        0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6], 
                              max_iter = 50000, cv = 10)
    elasticNet.fit(X_train, y_train)
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha )
    
    print("Try again for more precision with l1_ratio centered around " + str(ratio))
    elasticNet = ElasticNetCV(l1_ratio = [ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
                              alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6], 
                              max_iter = 50000, cv = 10)
    elasticNet.fit(X_train, y_train)
    if (elasticNet.l1_ratio_ > 1):
        elasticNet.l1_ratio_ = 1    
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha )
    
    print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio) + 
          " and alpha centered around " + str(alpha))
    elasticNet = ElasticNetCV(l1_ratio = ratio,
                              alphas = [alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85, alpha * .9, 
                                        alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3, 
                                        alpha * 1.35, alpha * 1.4], 
                              max_iter = 50000, cv = 10)
    elasticNet.fit(X_train, y_train)
    if (elasticNet.l1_ratio_ > 1):
        elasticNet.l1_ratio_ = 1    
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha )
    
    answer_generate(elasticNet, X_test, ids)
Example #3
0
print("Try again for more precision with l1_ratio centered around " +
      str(ratio))
elasticNet_model = ElasticNetCV(l1_ratio=[
    ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1,
    ratio * 1.15
],
                                alphas=[
                                    0.0001, 0.0003, 0.0006, 0.001, 0.003,
                                    0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1,
                                    3, 6
                                ],
                                max_iter=50000,
                                cv=10)
elasticNet_model.fit(dataset_train, y_train)
if (elasticNet_model.l1_ratio_ > 1):
    elasticNet_model.l1_ratio_ = 1
alpha = elasticNet_model.alpha_
ratio = elasticNet_model.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)

# version 3 :
print("Now try again for more precision on alpha, with l1_ratio fixed at " +
      str(ratio) + " and alpha centered around " + str(alpha))
elasticNet_model = ElasticNetCV(
    l1_ratio=ratio,
    alphas=[
        alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8,
        alpha * .85, alpha * .9, alpha * .95, alpha, alpha * 1.05, alpha * 1.1,
        alpha * 1.15, alpha * 1.25, alpha * 1.3, alpha * 1.35, alpha * 1.4
    ],
Example #4
0
def Elasticnet_regression(X=train_df_munged,Y=label_df['SalePrice']):
    elasticNet = ElasticNetCV(l1_ratio=[0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
                              alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
                                      0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
                              max_iter=50000, cv=10)
    print('handled data columns :\n', train_df_munged.columns)
    elasticNet.fit(X, Y)
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)
    print("Try again for more precision with l1_ratio centered around " + str(ratio))
    elasticNet = ElasticNetCV(
        l1_ratio=[ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
        alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
        max_iter=50000, cv=10)
    elasticNet.fit(X, Y)
    if (elasticNet.l1_ratio_ > 1):
        elasticNet.l1_ratio_ = 1
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)
    print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio) +
          " and alpha centered around " + str(alpha))
    elasticNet = ElasticNetCV(l1_ratio=ratio,
                              alphas=[alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85,
                                      alpha * .9,
                                      alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25,
                                      alpha * 1.3,
                                      alpha * 1.35, alpha * 1.4],
                              max_iter=50000, cv=10)
    elasticNet.fit(X, Y)
    if (elasticNet.l1_ratio_ > 1):
        elasticNet.l1_ratio_ = 1
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)

    y_train_ela = elasticNet.predict(X)

    # Plot residuals

    plt.scatter(y_train_ela, y_train_ela - Y, c="blue", marker="s", label="Training data")

    plt.title("Linear regression with ElasticNet regularization")
    plt.xlabel("Predicted values")
    plt.ylabel("Residuals")
    plt.legend(loc="upper left")
    plt.hlines(y=0, xmin=10.5, xmax=13.5, color="red")
    plt.show()
    # Plot predictions
    plt.scatter(Y, y_train_ela, c="blue", marker="s", label="Training data")

    plt.title("Linear regression with ElasticNet regularization")
    plt.xlabel("Predicted values")
    plt.ylabel("Real values")
    plt.legend(loc="upper left")
    plt.plot([10.5, 13.5], [10.5, 13.5], c="red")
    plt.show()
    return elasticNet
Example #5
0
print("Try again for more precision with l1_ratio centered around " +
      str(ratio))
elasticNet = ElasticNetCV(l1_ratio=[
    ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1,
    ratio * 1.15
],
                          alphas=[
                              0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
                              0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6
                          ],
                          max_iter=50000,
                          cv=10)
elasticNet.fit(X_train, y_train)
if (elasticNet.l1_ratio_ > 1):
    elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)

print("Now try again for more precision on alpha, with l1_ratio fixed at " +
      str(ratio) + " and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio=ratio,
                          alphas=[
                              alpha * .6, alpha * .65, alpha * .7, alpha * .75,
                              alpha * .8, alpha * .85, alpha * .9, alpha * .95,
                              alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15,
                              alpha * 1.25, alpha * 1.3, alpha * 1.35,
                              alpha * 1.4
                          ],
print("Try again for more precision with l1_ratio centered around " +
      str(ratio))
mod_enet = ElasticNetCV(l1_ratio=[
    ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1,
    ratio * 1.15
],
                        alphas=[
                            0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01,
                            0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6
                        ],
                        max_iter=50000,
                        cv=10)
mod_enet.fit(X_train, y_train)
if (mod_enet.l1_ratio_ > 1):
    mod_enet.l1_ratio_ = 1
alpha = mod_enet.alpha_
ratio = mod_enet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)

print("Now try again for more precision on alpha, with l1_ratio fixed at " +
      str(ratio) + " and alpha centered around " + str(alpha))
mod_enet = ElasticNetCV(l1_ratio=ratio,
                        alphas=[
                            alpha * .6, alpha * .65, alpha * .7, alpha * .75,
                            alpha * .8, alpha * .85, alpha * .9, alpha * .95,
                            alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15,
                            alpha * 1.25, alpha * 1.3, alpha * 1.35,
                            alpha * 1.4
                        ],
Example #7
0
def elasticNet(x_train, y_train, x_test, y_test):
    from sklearn.linear_model import ElasticNetCV

    # Setting the Hyper-parameters for ElasticNet
    elasticNet = ElasticNetCV(l1_ratio=[0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
                              alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
                                      0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
                              max_iter=50000, cv=10)

    elasticNet.fit(x_train, y_train)
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)

    # Tunnnig the Hyper-parameters for more precision
    print("Increasing the precision with l1_ratio centered around " + str(ratio))
    elasticNet = ElasticNetCV(l1_ratio=[ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05,
                                        ratio * 1.1, ratio * 1.15],
                              alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1,
                                      3, 6],
                              max_iter=50000, cv=10)

    elasticNet.fit(x_train, y_train)
    if (elasticNet.l1_ratio_ > 1):
        elasticNet.l1_ratio_ = 1

        # Printing the best alphas
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)

    print("Tunning for more precision on alpha, with l1_ratio fixed at " + str(ratio) +
          " and alpha centered around " + str(alpha))

    elasticNet = ElasticNetCV(l1_ratio=ratio,
                              alphas=[alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85,
                                      alpha * .9,
                                      alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25,
                                      alpha * 1.3,
                                      alpha * 1.35, alpha * 1.4],
                              max_iter=50000,
                              cv=10)

    elasticNet.fit(x_train, y_train)
    if (elasticNet.l1_ratio_ > 1):
        elasticNet.l1_ratio_ = 1

    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)

    y_train_ela = elasticNet.predict(x_train)
    y_test_ela = elasticNet.predict(x_test)

    # Plotting the residuals
    plt.scatter(y_train_ela, y_train_ela - y_train, c="blue", marker="s", label="Training data")
    plt.scatter(y_test_ela, y_test_ela - y_test, c="lightgreen", marker="s", label="Validation data")
    plt.title("Linear regression with ElasticNet regularization")
    plt.xlabel("Predicted values")
    plt.ylabel("Residuals")
    plt.legend(loc="upper left")
    plt.hlines(y=0, xmin=10.5, xmax=13.5, color="red")
    plt.show()

    # Plotting the predictions
    plt.scatter(y_train, y_train_ela, c="blue", marker="s", label="Training data")
    plt.scatter(y_test, y_test_ela, c="lightgreen", marker="s", label="Validation data")
    plt.title("Linear regression with ElasticNet regularization")
    plt.xlabel("Predicted values")
    plt.ylabel("Real values")
    plt.legend(loc="upper left")
    plt.plot([10.5, 13.5], [10.5, 13.5], c="red")
    plt.show()

    # Plotting the important coefficients
    coefs = pd.Series(elasticNet.coef_, index=x_train.columns)
    print("ElasticNet picked " + str(sum(coefs != 0)) + " features and eliminated the other " +
          str(sum(coefs == 0)) + " features")

    imp_coefs = pd.concat([coefs.sort_values().head(10),
                           coefs.sort_values().tail(10)])
    imp_coefs.plot(kind="barh")
    plt.title("Coefficients in the ElasticNet Model")
    plt.show()

    # Setting the Metric
    print("-" * 40)
    result = metric(y_test, y_test_ela)
Example #8
0
def elasticNet_model(data, y):
    X_train, X_test, y_train, y_test = train_test_split(data,
                                                        y,
                                                        test_size=0.3,
                                                        random_state=0)
    scorer = make_scorer(mean_squared_error, greater_is_better=False)

    def rmse_cv_train(model):
        rmse = np.sqrt(
            -cross_val_score(model, X_train, y_train, scoring=scorer, cv=10))
        return (rmse)

    def rmse_cv_test(model):
        rmse = np.sqrt(
            -cross_val_score(model, X_test, y_test, scoring=scorer, cv=10))
        return (rmse)

    elasticNet = ElasticNetCV(
        l1_ratio=[0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
        alphas=[
            0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1,
            0.3, 0.6, 1, 3, 6
        ],
        max_iter=50000,
        cv=10)
    elasticNet.fit(X_train, y_train)
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)

    print("Try again for more precision with l1_ratio centered around " +
          str(ratio))
    elasticNet = ElasticNetCV(l1_ratio=[
        ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1,
        ratio * 1.15
    ],
                              alphas=[
                                  0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
                                  0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6
                              ],
                              max_iter=50000,
                              cv=10)
    elasticNet.fit(X_train, y_train)
    if (elasticNet.l1_ratio_ > 1):
        elasticNet.l1_ratio_ = 1
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)

    print(
        "Now try again for more precision on alpha, with l1_ratio fixed at " +
        str(ratio) + " and alpha centered around " + str(alpha))
    elasticNet = ElasticNetCV(l1_ratio=ratio,
                              alphas=[
                                  alpha * .6, alpha * .65, alpha * .7,
                                  alpha * .75, alpha * .8, alpha * .85,
                                  alpha * .9, alpha * .95, alpha, alpha * 1.05,
                                  alpha * 1.1, alpha * 1.15, alpha * 1.25,
                                  alpha * 1.3, alpha * 1.35, alpha * 1.4
                              ],
                              max_iter=50000,
                              cv=10)
    elasticNet.fit(X_train, y_train)
    if (elasticNet.l1_ratio_ > 1):
        elasticNet.l1_ratio_ = 1
    alpha = elasticNet.alpha_
    ratio = elasticNet.l1_ratio_
    print("Best l1_ratio :", ratio)
    print("Best alpha :", alpha)

    print("ElasticNet RMSE on Training set :",
          rmse_cv_train(elasticNet).mean())
    print("ElasticNet RMSE on Test set :", rmse_cv_test(elasticNet).mean())
    y_train_ela = elasticNet.predict(X_train)
    y_test_ela = elasticNet.predict(X_test)

    # Plot residuals
    plt.scatter(y_train_ela,
                y_train_ela - y_train,
                c="blue",
                marker="s",
                label="Training data")
    plt.scatter(y_test_ela,
                y_test_ela - y_test,
                c="lightgreen",
                marker="s",
                label="Validation data")
    plt.title("Linear regression with ElasticNet regularization")
    plt.xlabel("Predicted values")
    plt.ylabel("Residuals")
    plt.legend(loc="upper left")
    plt.hlines(y=0, xmin=10.5, xmax=13.5, color="red")
    plt.show()

    # Plot predictions
    plt.scatter(y_train,
                y_train_ela,
                c="blue",
                marker="s",
                label="Training data")
    plt.scatter(y_test,
                y_test_ela,
                c="lightgreen",
                marker="s",
                label="Validation data")
    plt.title("Linear regression with ElasticNet regularization")
    plt.xlabel("Predicted values")
    plt.ylabel("Real values")
    plt.legend(loc="upper left")
    plt.plot([10.5, 13.5], [10.5, 13.5], c="red")
    plt.show()

    # Plot important coefficients
    coefs = pd.Series(elasticNet.coef_, index=X_train.columns)
    print("ElasticNet picked " + str(sum(coefs != 0)) +
          " features and eliminated the other " + str(sum(coefs == 0)) +
          " features")
    imp_coefs = pd.concat(
        [coefs.sort_values().head(10),
         coefs.sort_values().tail(10)])
    imp_coefs.plot(kind="barh")
    plt.title("Coefficients in the ElasticNet Model")
    plt.show()

    return elasticNet