def elastic(self):
elasticNet = ElasticNetCV(l1_ratio = [0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
elasticNet.fit(self.X_train, self.y_train)
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha )
print("Try again for more precision with l1_ratio centered around " + str(ratio))
elasticNet = ElasticNetCV(l1_ratio = [ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
elasticNet.fit(self.X_train, self.y_train)
# l1_ratio = 1 means elastic->l1_penalty
# l1_ratio = 0 means elastic->l2_penalty
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha )
print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio) +
" and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio = ratio,
alphas = [alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85, alpha * .9,
alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3,
alpha * 1.35, alpha * 1.4],
max_iter = 50000, cv = 10)
elasticNet.fit(self.X_train, self.y_train)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha )
print("ElasticNet RMSE on Training set :", rmse_cv_train(elasticNet).mean())
print("ElasticNet RMSE on Test set :", rmse_cv_test(elasticNet).mean())
def elastic_net(X_train, y_train, X_test, ids):
elasticNet = ElasticNetCV(l1_ratio = [0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
elasticNet.fit(X_train, y_train)
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha )
print("Try again for more precision with l1_ratio centered around " + str(ratio))
elasticNet = ElasticNetCV(l1_ratio = [ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
elasticNet.fit(X_train, y_train)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha )
print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio) +
" and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio = ratio,
alphas = [alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85, alpha * .9,
alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3,
alpha * 1.35, alpha * 1.4],
max_iter = 50000, cv = 10)
elasticNet.fit(X_train, y_train)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha )
answer_generate(elasticNet, X_test, ids)
print("Try again for more precision with l1_ratio centered around " +
str(ratio))
elasticNet_model = ElasticNetCV(l1_ratio=[
ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1,
ratio * 1.15
],
alphas=[
0.0001, 0.0003, 0.0006, 0.001, 0.003,
0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1,
3, 6
],
max_iter=50000,
cv=10)
elasticNet_model.fit(dataset_train, y_train)
if (elasticNet_model.l1_ratio_ > 1):
elasticNet_model.l1_ratio_ = 1
alpha = elasticNet_model.alpha_
ratio = elasticNet_model.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
# version 3 :
print("Now try again for more precision on alpha, with l1_ratio fixed at " +
str(ratio) + " and alpha centered around " + str(alpha))
elasticNet_model = ElasticNetCV(
l1_ratio=ratio,
alphas=[
alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8,
alpha * .85, alpha * .9, alpha * .95, alpha, alpha * 1.05, alpha * 1.1,
alpha * 1.15, alpha * 1.25, alpha * 1.3, alpha * 1.35, alpha * 1.4
],
def Elasticnet_regression(X=train_df_munged,Y=label_df['SalePrice']):
elasticNet = ElasticNetCV(l1_ratio=[0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter=50000, cv=10)
print('handled data columns :\n', train_df_munged.columns)
elasticNet.fit(X, Y)
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
print("Try again for more precision with l1_ratio centered around " + str(ratio))
elasticNet = ElasticNetCV(
l1_ratio=[ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter=50000, cv=10)
elasticNet.fit(X, Y)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio) +
" and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio=ratio,
alphas=[alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85,
alpha * .9,
alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25,
alpha * 1.3,
alpha * 1.35, alpha * 1.4],
max_iter=50000, cv=10)
elasticNet.fit(X, Y)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
y_train_ela = elasticNet.predict(X)
# Plot residuals
plt.scatter(y_train_ela, y_train_ela - Y, c="blue", marker="s", label="Training data")
plt.title("Linear regression with ElasticNet regularization")
plt.xlabel("Predicted values")
plt.ylabel("Residuals")
plt.legend(loc="upper left")
plt.hlines(y=0, xmin=10.5, xmax=13.5, color="red")
plt.show()
# Plot predictions
plt.scatter(Y, y_train_ela, c="blue", marker="s", label="Training data")
plt.title("Linear regression with ElasticNet regularization")
plt.xlabel("Predicted values")
plt.ylabel("Real values")
plt.legend(loc="upper left")
plt.plot([10.5, 13.5], [10.5, 13.5], c="red")
plt.show()
return elasticNet
print("Try again for more precision with l1_ratio centered around " +
str(ratio))
elasticNet = ElasticNetCV(l1_ratio=[
ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1,
ratio * 1.15
],
alphas=[
0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6
],
max_iter=50000,
cv=10)
elasticNet.fit(X_train, y_train)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
print("Now try again for more precision on alpha, with l1_ratio fixed at " +
str(ratio) + " and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio=ratio,
alphas=[
alpha * .6, alpha * .65, alpha * .7, alpha * .75,
alpha * .8, alpha * .85, alpha * .9, alpha * .95,
alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15,
alpha * 1.25, alpha * 1.3, alpha * 1.35,
alpha * 1.4
],
print("Try again for more precision with l1_ratio centered around " +
str(ratio))
mod_enet = ElasticNetCV(l1_ratio=[
ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1,
ratio * 1.15
],
alphas=[
0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01,
0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6
],
max_iter=50000,
cv=10)
mod_enet.fit(X_train, y_train)
if (mod_enet.l1_ratio_ > 1):
mod_enet.l1_ratio_ = 1
alpha = mod_enet.alpha_
ratio = mod_enet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
print("Now try again for more precision on alpha, with l1_ratio fixed at " +
str(ratio) + " and alpha centered around " + str(alpha))
mod_enet = ElasticNetCV(l1_ratio=ratio,
alphas=[
alpha * .6, alpha * .65, alpha * .7, alpha * .75,
alpha * .8, alpha * .85, alpha * .9, alpha * .95,
alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15,
alpha * 1.25, alpha * 1.3, alpha * 1.35,
alpha * 1.4
],
def elasticNet(x_train, y_train, x_test, y_test):
from sklearn.linear_model import ElasticNetCV
# Setting the Hyper-parameters for ElasticNet
elasticNet = ElasticNetCV(l1_ratio=[0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter=50000, cv=10)
elasticNet.fit(x_train, y_train)
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
# Tunnnig the Hyper-parameters for more precision
print("Increasing the precision with l1_ratio centered around " + str(ratio))
elasticNet = ElasticNetCV(l1_ratio=[ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05,
ratio * 1.1, ratio * 1.15],
alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1,
3, 6],
max_iter=50000, cv=10)
elasticNet.fit(x_train, y_train)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
# Printing the best alphas
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
print("Tunning for more precision on alpha, with l1_ratio fixed at " + str(ratio) +
" and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio=ratio,
alphas=[alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85,
alpha * .9,
alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25,
alpha * 1.3,
alpha * 1.35, alpha * 1.4],
max_iter=50000,
cv=10)
elasticNet.fit(x_train, y_train)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
y_train_ela = elasticNet.predict(x_train)
y_test_ela = elasticNet.predict(x_test)
# Plotting the residuals
plt.scatter(y_train_ela, y_train_ela - y_train, c="blue", marker="s", label="Training data")
plt.scatter(y_test_ela, y_test_ela - y_test, c="lightgreen", marker="s", label="Validation data")
plt.title("Linear regression with ElasticNet regularization")
plt.xlabel("Predicted values")
plt.ylabel("Residuals")
plt.legend(loc="upper left")
plt.hlines(y=0, xmin=10.5, xmax=13.5, color="red")
plt.show()
# Plotting the predictions
plt.scatter(y_train, y_train_ela, c="blue", marker="s", label="Training data")
plt.scatter(y_test, y_test_ela, c="lightgreen", marker="s", label="Validation data")
plt.title("Linear regression with ElasticNet regularization")
plt.xlabel("Predicted values")
plt.ylabel("Real values")
plt.legend(loc="upper left")
plt.plot([10.5, 13.5], [10.5, 13.5], c="red")
plt.show()
# Plotting the important coefficients
coefs = pd.Series(elasticNet.coef_, index=x_train.columns)
print("ElasticNet picked " + str(sum(coefs != 0)) + " features and eliminated the other " +
str(sum(coefs == 0)) + " features")
imp_coefs = pd.concat([coefs.sort_values().head(10),
coefs.sort_values().tail(10)])
imp_coefs.plot(kind="barh")
plt.title("Coefficients in the ElasticNet Model")
plt.show()
# Setting the Metric
print("-" * 40)
result = metric(y_test, y_test_ela)
def elasticNet_model(data, y):
X_train, X_test, y_train, y_test = train_test_split(data,
y,
test_size=0.3,
random_state=0)
scorer = make_scorer(mean_squared_error, greater_is_better=False)
def rmse_cv_train(model):
rmse = np.sqrt(
-cross_val_score(model, X_train, y_train, scoring=scorer, cv=10))
return (rmse)
def rmse_cv_test(model):
rmse = np.sqrt(
-cross_val_score(model, X_test, y_test, scoring=scorer, cv=10))
return (rmse)
elasticNet = ElasticNetCV(
l1_ratio=[0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
alphas=[
0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1,
0.3, 0.6, 1, 3, 6
],
max_iter=50000,
cv=10)
elasticNet.fit(X_train, y_train)
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
print("Try again for more precision with l1_ratio centered around " +
str(ratio))
elasticNet = ElasticNetCV(l1_ratio=[
ratio * .85, ratio * .9, ratio * .95, ratio, ratio * 1.05, ratio * 1.1,
ratio * 1.15
],
alphas=[
0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6
],
max_iter=50000,
cv=10)
elasticNet.fit(X_train, y_train)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
print(
"Now try again for more precision on alpha, with l1_ratio fixed at " +
str(ratio) + " and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio=ratio,
alphas=[
alpha * .6, alpha * .65, alpha * .7,
alpha * .75, alpha * .8, alpha * .85,
alpha * .9, alpha * .95, alpha, alpha * 1.05,
alpha * 1.1, alpha * 1.15, alpha * 1.25,
alpha * 1.3, alpha * 1.35, alpha * 1.4
],
max_iter=50000,
cv=10)
elasticNet.fit(X_train, y_train)
if (elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
print("Best alpha :", alpha)
print("ElasticNet RMSE on Training set :",
rmse_cv_train(elasticNet).mean())
print("ElasticNet RMSE on Test set :", rmse_cv_test(elasticNet).mean())
y_train_ela = elasticNet.predict(X_train)
y_test_ela = elasticNet.predict(X_test)
# Plot residuals
plt.scatter(y_train_ela,
y_train_ela - y_train,
c="blue",
marker="s",
label="Training data")
plt.scatter(y_test_ela,
y_test_ela - y_test,
c="lightgreen",
marker="s",
label="Validation data")
plt.title("Linear regression with ElasticNet regularization")
plt.xlabel("Predicted values")
plt.ylabel("Residuals")
plt.legend(loc="upper left")
plt.hlines(y=0, xmin=10.5, xmax=13.5, color="red")
plt.show()
# Plot predictions
plt.scatter(y_train,
y_train_ela,
c="blue",
marker="s",
label="Training data")
plt.scatter(y_test,
y_test_ela,
c="lightgreen",
marker="s",
label="Validation data")
plt.title("Linear regression with ElasticNet regularization")
plt.xlabel("Predicted values")
plt.ylabel("Real values")
plt.legend(loc="upper left")
plt.plot([10.5, 13.5], [10.5, 13.5], c="red")
plt.show()
# Plot important coefficients
coefs = pd.Series(elasticNet.coef_, index=X_train.columns)
print("ElasticNet picked " + str(sum(coefs != 0)) +
" features and eliminated the other " + str(sum(coefs == 0)) +
" features")
imp_coefs = pd.concat(
[coefs.sort_values().head(10),
coefs.sort_values().tail(10)])
imp_coefs.plot(kind="barh")
plt.title("Coefficients in the ElasticNet Model")
plt.show()
return elasticNet
