def get_ridge_regression_weights(X, Y, alphas):
ridge = Ridge(normalize=True)
coefs = []
for a in alphas:
ridge.set_params(alpha=a)
ridge.fit(X, Y)
coefs.append(ridge.coef_)
ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
plt.axis('tight')
plt.xlabel('alpha')
plt.ylabel('weights')
plt.plot(alphas, coefs)
plt.title("Weight of Features as Alpha Increases")
plt.show()
for a in alphas:
X_train, X_test, y_train, y_test = train_test_split(X,
Y,
test_size=0.5,
random_state=1)
ridge = Ridge(alpha=a, normalize=True)
ridge.fit(X_train, y_train)
pred2 = ridge.predict(X_test)
print("alpha: ", a)
print("Ridge Coeffecients")
print(pd.Series(ridge.coef_, index=X.columns))
print("MSE: " + str(mean_squared_error(y_test, pred2)))
print("*******************************")
def Ridge_model(train_linear, test_linear):
ridgecv = RidgeCV(alphas = np.logspace(-5, 4, 400))
ridgecv.fit(train_linear_fea, train_linear_tar)
ridgecv_score = ridgecv.score(train_linear_fea, train_linear_tar)
ridgecv_alpha = ridgecv.alpha_
print("Best alpha : ", ridgecv_alpha, "Score: ",ridgecv_score)
coef=pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending =False)
start=time.time()
ridge =Ridge(normalize = True)
ridge.set_params(alpha=ridgecv_alpha,max_iter = 10000)
#ridge.set_params(alpha=6,max_iter = 10000)
ridge.fit(x_train, y_train)
end=time.time()
mean_squared_error(y_test, ridge.predict(x_test))
coef_ridge=pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending =False)
evaluate(ridge,x_test,y_test,x_train,y_train)
print('Time elapsed: %.4f seconds' % (end-start))
y_ridge_predict=ridge.predict(train_linear_fea)
x_line = np.arange(700000)
y_line=x_line
plt.scatter(real_train_tar,np.expm1(y_ridge_predict))
plt.plot(x_line, y_line, color='r')
plt.xlabel('Actual Sale Price')
plt.ylabel('Predict Sle Price')
test_prediction_ridge=np.expm1(ridge.predict(test_linear))
write_pkl(ridgecv_alpha, '/Users/vickywinter/Documents/NYC/Machine Learning Proj/Pickle/ridge_params.pkl')
return test_prediction_ridge
def create_reg(self):
if self.linear_model == 'linear_regression':
return LinearRegression()
elif self.linear_model == 'ridge':
model = Ridge(self.alpha)
model.set_params(**self.params)
return model
def Ridge_model(train_linear, test_linear):
ridgecv = RidgeCV(alphas=np.logspace(-5, 4, 400))
ridgecv.fit(train_linear_fea, train_linear_tar)
ridgecv_score = ridgecv.score(train_linear_fea, train_linear_tar)
ridgecv_alpha = ridgecv.alpha_
print("Best alpha : ", ridgecv_alpha, "Score: ", ridgecv_score)
coef = pd.Series(ridgecv.coef_,
index=x_train.columns).sort_values(ascending=False)
start = time.time()
ridge = Ridge(normalize=True)
ridge.set_params(alpha=ridgecv_alpha, max_iter=10000)
#ridge.set_params(alpha=6,max_iter = 10000)
ridge.fit(x_train, y_train)
end = time.time()
mean_squared_error(y_test, ridge.predict(x_test))
coef_ridge = pd.Series(ridgecv.coef_,
index=x_train.columns).sort_values(ascending=False)
evaluate(ridge, x_test, y_test, x_train, y_train)
print('Time elapsed: %.4f seconds' % (end - start))
y_ridge_predict = ridge.predict(train_linear_fea)
x_line = np.arange(700000)
y_line = x_line
plt.scatter(real_train_tar, np.expm1(y_ridge_predict))
plt.plot(x_line, y_line, color='r')
plt.xlabel('Actual Sale Price')
plt.ylabel('Predict Sle Price')
test_prediction_ridge = np.expm1(ridge.predict(test_linear))
write_pkl(
ridgecv_alpha,
'/Users/vickywinter/Documents/NYC/Machine Learning Proj/Pickle/ridge_params.pkl'
)
return test_prediction_ridge
def linear_ridge2():
X = 1. / (np.arange(1, 11) + np.arange(0, 10)[:, np.newaxis])
y = np.ones(10)
n_alphas = 200
# alphas count is 200, 都在10的-10次方和10的-2次方之间
alphas = np.logspace(-10, -2, n_alphas)
clf = Ridge(fit_intercept=False)
coefs = []
# 循环200次
for a in alphas:
# 设置本次循环的超参数
clf.set_params(alpha=a)
# 针对每个alpha做ridge回归
clf.fit(X, y)
# 把每一个超参数alpha对应的theta存下来
coefs.append(clf.coef_)
ax = plt.gca()
ax.plot(alphas, coefs)
# 将alpha的值取对数便于画图
ax.set_xscale('log')
# 翻转x轴的大小方向，让alpha从大到小显示
ax.set_xlim(ax.get_xlim()[::-1])
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Ridge coefficients as a function of the regularization')
plt.axis('tight')
plt.show()
def ridge_reg():
n_alphas = 200
ridge_alphas = np.logspace(-2, 6, n_alphas)
ridge_cv = RidgeCV(alphas=ridge_alphas,
scoring='neg_mean_squared_error',
cv=3)
ridge_cv.fit(X, y)
print('------------')
print(ridge_cv.coef_, "\n", ridge_cv.alpha_)
# Выведем влияние альфы (от наибольшей к наименьшей) для каждого признака
model = Ridge()
coefs = []
for a in ridge_alphas:
model.set_params(alpha=a)
model.fit(X, y)
coefs.append(model.coef_)
ax = plt.gca()
ax.plot(ridge_alphas, coefs)
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1]) # reverse axis
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Ridge coefficients as a function of the regularization')
plt.axis('tight')
plt.show()
def _ridge_search(X,
Y,
train,
test,
alphas=np.logspace(0, 4, 20),
scoring=corr):
"""Fit ridge regression sweeping through alphas
Parameters
----------
X : array (n_samples, n_predictors)
design matrix/predictors
Y : array (n_samples, n_features)
response matrix
train : array
index array for training
test : array
index array for testing
alphas : array
alpha values used to fit
scoring : callable
function used to score the fit (default correlation)
Returns
-------
scores : array (n_alphas, n_features)
score of prediction for each alpha and each feature
"""
ridge = Ridge(fit_intercept=False, solver='svd')
scores = []
for alpha in alphas:
ridge.set_params(alpha=alpha)
ridge.fit(X[train], Y[train])
scores.append(scoring(Y[test].T, ridge.predict(X[test]).T))
return np.array(scores)
def fit_data(X, Y):
print("Fitting started...")
from sklearn.linear_model import Ridge, RidgeCV, Lasso, LassoCV
print("Least Square")
coef = np.linalg.lstsq(X, Y, rcond=None)[0]
print(coef)
#print(intercept)
print("Ridge")
#ridgecv = RidgeCV(alphas = None, cv = 10)
#ridgecv.fit(X, Y)
rr100 = Ridge(alpha=100)
rr100.fit(X, Y)
print(rr100.coef_)
print(rr100.intercept_)
print("LASSO")
lassocv = LassoCV(alphas = None, cv = 10, max_iter = 100000)
lassocv.fit(X, Y)
rr100 = Lasso(max_iter=10000) # comparison with alpha value
rr100.set_params(alpha=lassocv.alpha_)
rr100.fit(X, Y)
print(rr100.coef_)
print(rr100.intercept_)
print("")
def load_regressor_Ridge(X_train, y_train):
best_estimator_ridge = joblib.load('./built_models/regressor_ridge.pkl')
regressor_ridge = Ridge()
regressor_ridge.set_params(**best_estimator_ridge)
print regressor_ridge
regressor_ridge.fit(X_train, y_train)
return regressor_ridge
def ridge_coefs(X, y, alphas):
coefs = []
ridge_reg = Ridge()
for a in alphas:
ridge_reg.set_params(alpha=a)
ridge_reg.fit(X, y)
coefs.append(ridge_reg.coef_[0])
return coefs
def non_zero_coeff(X, y, _lambda):
clf = Ridge(normalize=True, solver='sag')
# clf = Lasso()
clf.set_params(alpha=_lambda)
clf.fit(X, y)
nonzero = np.count_nonzero(clf.coef_)
print("Number of non-zero features: " + str(nonzero) + ", alpha: " +
str(_lambda))
return nonzero, (clf.coef_ != 0)
def make_ridge_pred(df, next_week, debug=0):
"""
This method creates predictions using ridge regression.
"""
#Tuned##
params_old = {
'alpha': 10,
'max_iter': 1,
'solver': 'cholesky',
'normalize': True,
'fit_intercept': True
}
params = {
'alpha': 10,
'max_iter': -1,
'solver': 'sparse_cg',
'normalize': False,
'fit_intercept': True
}
rand_space = {
'alpha': [1e-10, 1e-5, 1e-2, 1e-1, 1, 10, 100],
'normalize': [True, False],
'fit_intercept': [True, False],
'solver':
['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'],
'max_iter': [int(x) for x in np.linspace(1, 1000, num=10)]
}
space = {
'alpha': [8, 9, 10, 11],
'normalize': [True, False],
'fit_intercept': [True, False],
'solver': ['cholesky'],
'max_iter': [None, 1, 2, 10]
}
X_train, X_test, Y_train, Y_test = process_data(df, next_week)
multi_ridge = Ridge()
multi_ridge.set_params(**params)
#best_random = random_search(multi_ridge, rand_space, next_week, 100, 3, X_train, Y_train)
#best_random = grid_search(multi_ridge, space, next_week, 3, X_train, Y_train)
multi_ridge.fit(X_train, Y_train)
next_week[Y_train.columns] = multi_ridge.predict(
next_week[X_train.columns])
if debug:
y_pred_untrain = multi_ridge.predict(X_train)
print(next_week)
print("Score: ", multi_ridge.score(X_train, Y_train) * 100)
print("MSE: ", metrics.mean_squared_error(Y_train, y_pred_untrain))
print(
"CV: ",
ms.cross_val_score(multi_ridge,
Y_train,
y_pred_untrain,
cv=10,
scoring='neg_mean_squared_error'))
return next_week
class Ridge(Model):
def create_model(self):
self.ridge = RidgeLR()
def fit(self, train_x, train_y):
self.ridge.fit(train_x, train_y)
def set_config(self, config):
self.ridge.set_params(**config)
def predict(self, test_x):
return self.ridge.predict(test_x)
def plot_regularizations(X_train, X_test, y_train, y_test, alphas):
# Compute train and test errors
regressor = Ridge() # regularized linear regression
train_errors = list()
test_errors = list()
for alpha in alphas:
print("Woring on alpha = %s..." % alpha)
regressor.set_params(alpha=alpha)
regressor.fit(X_train, y_train)
train_error = regressor.score(X_train, y_train)
print("train_error : %s..." % train_error)
train_errors.append(train_error)
test_error = regressor.score(X_test, y_test)
print("test_error : %s..." % test_error)
test_errors.append(test_error)
i_alpha_optim = np.argmax(test_errors)
alpha_optim = alphas[i_alpha_optim]
print("Optimal regularization parameter : %s" % alpha_optim)
# Estimate the coef_ on full data with optimal regularization parameter
regressor.set_params(alpha=alpha_optim)
coef_ = regressor.fit(X_train, y_train).coef_
# #############################################################################
# Plot results functions
import matplotlib.pyplot as plt
plt.subplot(2, 1, 1)
plt.semilogx(alphas, train_errors, label='Train')
plt.semilogx(alphas, test_errors, label='Test')
plt.vlines(alpha_optim,
plt.ylim()[0],
np.max(test_errors),
color='k',
linewidth=3,
label='Optimum on test')
plt.legend(loc='lower left')
plt.ylim([0, 1.2])
plt.xlabel('Regularization parameter')
plt.ylabel('Performance')
# Show estimated coef_ vs true coef
plt.subplot(2, 1, 2)
plt.plot(coef_, label='Estimated coef')
plt.legend()
plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.26)
plt.show()
def LinearModelRidge(X_train, y_train, X_test, y_test):
alphas = 10**np.linspace(10, -2, 100) * 0.5
ridgecv = RidgeCV(alphas=alphas, scoring="neg_mean_squared_error", cv=10)
ridgecv.fit(X_train, y_train)
print("Value of lambda ", ridgecv.alpha_)
ridge = Ridge()
ridge.set_params(alpha=ridgecv.alpha_)
ridge.fit(X_train, y_train)
print_evaluation_metrics(ridge, "Ridge Model", X_train, y_train, X_test,
y_test)
def feature_selection(df,
target,
test_split,
α=5,
thresh=50): ### Bich-Tien PHAN
"""
:param df: (pandas DataFrame) data to process
:param target: (string) label name to predict
:param test_split: (float) percentage of test data
:param α: (float) ridge regression hyperparameter, the maximal value to determine the optimal value of alpha for ridge regression
:param thresh: (float) parameter to select the significant features
:return: list of colums to keep after ridge regression
"""
#Prepocessing for ridge regression
X = df.drop(columns=target)
X = preprocessing.scale(X)
y = df[target].to_numpy()
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=test_split)
coefs = []
error = []
ridge_reg = Ridge()
n_alphas = 200
alphas = np.logspace(-5, α, n_alphas)
# iterate lambdas
for alpha in alphas:
# training
ridge_reg.set_params(alpha=alpha)
ridge_reg.fit(X_train, y_train)
coefs.append(ridge_reg.coef_)
error.append([alpha, np.mean((ridge_reg.predict(X_test) - y_test)**2)])
final_alpha = min(error, key=lambda t: t[1])[0]
ridge_reg.set_params(alpha=final_alpha)
ridge_reg.fit(X_train, y_train)
# Here we select the features based on its coefficient determined from ridge regression in comparison to the highest absolute value of coefficient.
# The threshold determines the features considered.
abs_coef = abs(ridge_reg.coef_)
abs_coef_percent = abs_coef / max(
abs_coef
) * 100 # Percentage vis-a-vis the highest absolute coefficient
get_features = np.where(abs_coef_percent > thresh)[0]
col_df = list(df.columns)
list_features = [col_df[i] for i in get_features if col_df[i] != target]
return list_features
def ridge_regr(train, test):
train_normalized = train / train.std()
test_normalized = test / test.std()
heat_ridge = Ridge()
a = 1e0
heat_ridge.set_params(alpha=a)
heat_ridge.fit(train_normalized[['Latitude', 'LoadEntering', 'Longitude']],
train_normalized.Capacity)
y_predict = heat_ridge.predict(
test_normalized[['Latitude', 'LoadEntering', 'Longitude']])
filename = "ridge_result.sav"
object_loc = open(filename, 'wb')
pickle.dump(heat_ridge, object_loc)
object_loc.close()
return filename, y_predict
def rr_regress(X, y, a=None, b=None):
"""
This function returns regression of input data by Ridge regression
method.
Parameters
----------
X: an array or array-like predictors.
It should be scaled by StandardScaler.
y: an array or array-like target.
It should has compatible dimension with input X.
a, b: an array or array-like, optional.
another set of data, such as a = X_test, b = y_test.
Returns
-------
coefs_RR: list.
a list of coefficients from RR with different lambdas
lambdas_RR: list.
a list of lambdas used in this RR
error1_RR: list.
a list of MSE of prediction from first input set (X, y)
error2_RR: list.
a list of MSE of prediction from second input set (a, b).
Return as None if a and b are not defined.
modelRR: modelRR = Ridge(), the LASSO model command
"""
# RR vs lambda
coefs_RR = []
error1_RR = []
error2_RR = []
# Tunning parameter(lambda)
lambdas_RR = np.logspace(-4, 8, 200)
modelRR = Ridge()
# loop over lambda values to determine the best by mse
for l in lambdas_RR:
modelRR.set_params(alpha=l)
modelRR.fit(X, y)
coefs_RR.append(modelRR.coef_)
error1_RR.append(mean_squared_error(y, modelRR.predict(X)))
if a.any() and b.any() is not None:
error2_RR.append(mean_squared_error(b, modelRR.predict(a)))
else:
error2_RR = None
return coefs_RR, lambdas_RR, error1_RR, error2_RR, modelRR
def ridge_with_single_search():
data = init_data()
X = data[:, 1:]
Y = data[:, 0]
X_train, X_test, y_train, y_test = train_test_split(X,
Y,
test_size=0.3,
random_state=42)
alphas = [0, 0.03, 0.05, 0.1, 0.2, 0.4, 0.5, 0.6, 1, 1.5, 2]
_ridge = Ridge()
for a in alphas:
_ridge.set_params(alpha=a)
_ridge.fit(X_train, y_train)
test_y_pred = _ridge.predict(X_test)
print("alpha is %f,测试集MSE is %f" %
(a, mean_squared_error(y_test, test_y_pred)))
def Ridge_Reg(Phi_train, Y_train, Phi_test, Y_test, alphas):
reg = Ridge()
coefs = []
train_MSE = []
test_MSE = []
for a in alphas:
reg.set_params(alpha=a)
reg.fit(Phi_train, Y_train)
coefs.append(reg.coef_)
train_pred = reg.predict(Phi_train)
train_MSE.append(mean_squared_error(Y_train, train_pred))
test_pred = reg.predict(Phi_test)
test_MSE.append(mean_squared_error(Y_test, test_pred))
return {'coefs': coefs, 'train_MSE': train_MSE, 'test_MSE': test_MSE}
def ridge_cv(k, X, y, alphas):
splits = k_split(k, X.shape[0])
results = []
for alpha in alphas:
temp_ridges = []
for i, split in enumerate(splits):
X_train = X.iloc[split[1]]
y_train = y.iloc[split[1]]
X_test = X.iloc[split[0]]
y_test = y.iloc[split[0]]
ridge = Ridge(normalize=True)
ridge.set_params(alpha=alpha)
ridge.fit(X_train, y_train)
predicted_y = ridge.predict(X_test)
temp_ridges.append(mean_squared_error(y_true=y_test, y_pred=predicted_y))
results.append(sum(temp_ridges)/len(temp_ridges))
return results
def best_param(tipoReg, alphas):
print tipoReg
Xm = Xtrain.as_matrix()
ym = ytrain.as_matrix()
k_fold = cross_validation.KFold(len(Xm),10)
best_cv_mse = float("inf")
if tipoReg == "Ridge":
modelo = Ridge(fit_intercept = True)
elif tipoReg == "Lasso":
modelo = Lasso(fit_intercept = True)
for a in alphas:
modelo.set_params(alpha=a)
mse_list_k10 = [
MSE(modelo.fit(Xm[train], ym[train]).predict(Xm[val]), ym[val])
for train, val in k_fold]
if np.mean(mse_list_k10) < best_cv_mse:
best_cv_mse = np.mean(mse_list_k10)
best_alpha = a
print "BEST PARAMETER=%f, MSE(CV)=%f"%(best_alpha, best_cv_mse)
def best_param(tipoReg, alphas):
print tipoReg
Xm = Xtrain.as_matrix()
ym = ytrain.as_matrix()
k_fold = cross_validation.KFold(len(Xm), 10)
best_cv_mse = float("inf")
if tipoReg == "Ridge":
modelo = Ridge(fit_intercept=True)
elif tipoReg == "Lasso":
modelo = Lasso(fit_intercept=True)
for a in alphas:
modelo.set_params(alpha=a)
mse_list_k10 = [
MSE(modelo.fit(Xm[train], ym[train]).predict(Xm[val]), ym[val])
for train, val in k_fold
]
if np.mean(mse_list_k10) < best_cv_mse:
best_cv_mse = np.mean(mse_list_k10)
best_alpha = a
print "BEST PARAMETER=%f, MSE(CV)=%f" % (best_alpha, best_cv_mse)
def get_ridge(X, Y):
#running ridge with alpha 0 (MLR)
ridge = Ridge()
ridge.set_params(alpha=0, normalize=True)
ridge.fit(X, Y)
#Grid search for ridge
alphas_ridge = np.linspace(0, 15, 50)
tuned_parameters_r = [{'alpha': alphas_ridge}]
n_folds = 5
cv = KFold(n_splits=n_folds, shuffle=True)
tune_ridge = GridSearchCV(ridge,
tuned_parameters_r,
cv=cv,
refit=True,
return_train_score=True,
scoring='neg_mean_squared_error')
tune_ridge.fit(X, Y)
print(tune_ridge.best_params_)
print(np.max(tune_ridge.cv_results_['mean_test_score']))
print(np.min(tune_ridge.cv_results_['mean_test_score']))
ridge_best = tune_ridge.best_estimator_
ridge_best.fit(X, Y)
print(ridge_best.score(X, Y))
suffix = str(datetime.datetime.now())
model_filename = 'ridge' + suffix + '.sav'
pickle.dump(ridge_best, open(model_filename, 'wb'))
csv_filename = 'ridge ' + suffix + '.csv'
raw_test, test_IDs = load_test()
predict = ridge_best.predict(raw_test)
predict = np.exp(predict)
predict = pd.DataFrame(predict)
predict = pd.concat([test_IDs, predict], axis=1)
predict.columns = ['Id', 'SalePrice']
predict.to_csv(csv_filename, index=False)
def test_regressor_modifications(self):
regressor = Ridge(alpha=1e-8)
pcovr = self.model(mixing=0.5, regressor=regressor)
# PCovR regressor matches the original
self.assertTrue(regressor.get_params() == pcovr.regressor.get_params())
# PCovR regressor updates its parameters
# to match the original regressor
regressor.set_params(alpha=1e-6)
self.assertTrue(regressor.get_params() == pcovr.regressor.get_params())
# Fitting regressor outside PCovR fits the PCovR regressor
regressor.fit(self.X, self.Y)
self.assertTrue(hasattr(pcovr.regressor, "coef_"))
# PCovR regressor doesn't change after fitting
pcovr.fit(self.X, self.Y)
regressor.set_params(alpha=1e-4)
self.assertTrue(hasattr(pcovr.regressor_, "coef_"))
self.assertTrue(
regressor.get_params() != pcovr.regressor_.get_params())
nb_runs_cv = len(x)
logo = LeaveOneOut() # leave on run out !
cv_index = 0
scores = np.zeros((nb_voxels, nb_runs_cv, nb_alphas))
for train, valid in logo.split(y):
y_train = [
y[i] for i in train
] # fmri_runs liste 2D colonne = voxels et chaque row = un t_i
x_train = [x[i] for i in train]
dm = np.vstack(x_train)
fmri = np.vstack(y_train)
alpha_index = 0
for alpha_tmp in alphas: # compute the r2 for a given alpha for all the voxel
model.set_params(alpha=alpha_tmp)
model_fitted = model.fit(dm, fmri)
r2 = get_r2_score(model_fitted, y[valid[0]], x[valid[0]])
scores[:, cv_index, alpha_index] = r2
alpha_index += 1
cv_index += 1
best_alphas_indexes = np.argmax(np.mean(scores, axis=1), axis=1)
voxel2alpha = np.array([alphas[i] for i in best_alphas_indexes])
# compute best alpha for each voxel and group them by alpha-value
alpha2voxel = {key: [] for key in alphas}
for index in range(len(voxel2alpha)):
alpha2voxel[voxel2alpha[index]].append(index)
for alpha in alphas:
yaml_path = os.path.join(args.output,
'run_{}_alpha_{}.yml'.format(run, alpha))
# In[56]: #linear regression no regularization reg = linear_model.Ridge(alpha=0) reg = reg.fit(x_nn, y_nn) y_predicted = reg.predict(x_nn) plt.figure() plt.plot(time_axis[30:66], y_predicted, 'r-') plt.plot(x_test, yn_test, 'ko', label="Original Noised Data") plt.show() # In[ ]: #linear regression using regularization alpha = 10000000 clf = Ridge() clf.set_params(alpha=100000000) regression = clf.fit(x_nn, y_nn) y_predicted_2 = regression.predict(x_nn) plt.figure() plt.plot(time_axis[30:66], y_predicted_2, 'r-') plt.plot(x_test, yn_test, 'ko', label="Original Noised Data") plt.show() # # Neural networks # # Class 1 # In[53]: model_class1 = get_model_for_Class(18) model_class1.fit(x=x_nn, y=y_nn, epochs=100, batch_size=1)
X = X.drop('intercept', axis=1)
Xtrain = X[istrain]
ytrain = y[istrain]
names_regressors = ["Lcavol", "Lweight", "Age", "Lbph", "Svi", "Lcp", "Gleason", "Pgg45"]
alphas_ = np.logspace(4,-1,base=10)
Xm = Xtrain.as_matrix()
ym = ytrain.as_matrix()
k_fold = cross_validation.KFold(len(Xm),10)
best_cv_mse = float("inf")
model = Ridge(fit_intercept=True,solver='svd')
for a in alphas_:
model.set_params(alpha=a)
mse_list_k10 = [MSE(model.fit(Xm[train], ym[train]).predict(Xm[val]), ym[val]) for train, val in k_fold]
if np.mean(mse_list_k10) < best_cv_mse:
best_cv_mse = np.mean(mse_list_k10)
best_alpha = a
print "RIDGE BEST PARAMETER=%f, MSE(CV)=%f"%(best_alpha,best_cv_mse)
ridge_model.intercept_
# In[23]:
# linspace(10,-2,100) 10 ile -2 arasında 100 değer
lambdalar = 10**np.linspace(10, -2, 100) * 0.5
lambdalar
# In[26]:
ridge_model = Ridge()
katsayilar = []
for i in lambdalar:
ridge_model.set_params(alpha=i)
ridge_model.fit(X_train, Y_train)
katsayilar.append(ridge_model.coef_)
katsayilar
# In[27]:
ax = plt.gca()
ax.plot(lambdalar, katsayilar)
# katsayiların hepsini gözlemleyebilmek için rakamları birbirine
# yakınlaştıralım deyip düzleştirme yapıyoruz
ax.set_xscale("log")
# renkler katsayı değerleri
# x ekseni lambda değerleri
X['Student'] = X['Student'].map({'Yes': 1, 'No': 0})
X['Married'] = X['Married'].map({'Yes': 1, 'No': 0})
X = X.drop('Ethnicity', axis=1)
y = Credit['Balance'].values
# X.head() if you'd like to look at the data
# first without normalization
n_alphas = 200
alphas = np.logspace(-1, 4, n_alphas)
clf = Ridge(normalize=False)
coef_names = list(X.columns)
coefs = []
for a in alphas:
clf.set_params(alpha=a, normalize=False)
clf.fit(X.values, y)
coefs.append(clf.coef_)
ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Ridge coefficients without Normalization')
plt.axis('tight')
plt.legend(coef_names)
plt.show()
# now with normalization
X = preprocessing.scale(X)
X_train, X_test, y_train, y_test = cross_validation.train_test_split(
X, Y, test_size=0.2)
clf = Ridge()
#clf.fit(X_train, y_train)
alpha_ridge = [0, 0.1, 1, 10, 100, 1000, 10000, 100000]
errors_test = []
errors_train = []
L2Norm_weights = []
weights = []
for i in alpha_ridge:
clf.set_params(alpha=i)
clf.fit(X_train, y_train)
Y_pred = clf.predict(X_test)
errors_test.append(sqrt(mean_squared_error(y_test, Y_pred)))
Y_pred2 = clf.predict(X_train)
errors_train.append(sqrt(mean_squared_error(y_train, Y_pred2)))
L2Norm_weights.append((np.linalg.norm(clf.coef_)))
weights.append(clf.coef_)
weights = array(weights)
weights = np.matrix.transpose(weights)
#************************ PART (A) AND (B)************************************
n_features = X.shape[1]
params = {'max_features':['auto','sqrt','log2']}
RF_model = GridSearchCV(RF_est, params)
RF_model.fit(X,y)
print('Best {}'.format(RF_model.best_params_))
print('Performing grid search on GBR')
n_features = X.shape[1]
params = {'max_features':['auto','sqrt','log2'],
'max_depth':[2, 3]}
GBR_model = GridSearchCV(GBR_est, params)
GBR_model.fit(X,y)
print('Best {}'.format(GBR_model.best_params_))
else:
Lin_model = Lin_est.set_params(alpha=100.0)
SVR_model = svr_est.set_params(C=1.0)
RF_model = RF_est.set_params(max_features='auto')
GBR_model = GBR_est.set_params(max_features='auto',
max_depth=3)
#%% Specify set of models to test
model_set = [('Null',LCM.rand_pick_mod()),
('Lin', Lin_model),
('Lin_SVR',SVR_model),
('GBR',GBR_model),
('RF', RF_model)]
# model_set = [('Null',LCM.rand_pick_mod()),
# ('Lin', Lin_model),
# ('RF', RF_model)]
def Model(train_linear, test_linear):
train_linear_fea=train_linear.drop(columns=['SalePrice'])
train_linear_tar=train_linear.SalePrice
x_train, x_test, y_train, y_test = train_test_split(train_linear_fea, train_linear_tar,test_size=0.2, random_state=0)
def evaluate(model, test_features, test_labels,train_features, train_labels):
predictions = model.predict(test_features)
errors = abs(predictions - test_labels)
mape = 100 * np.mean(errors / test_labels)
accuracy = 100 - mape
print('Model Performance')
print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
print('Accuracy = {:0.2f}%.'.format(accuracy))
print("MSE for train data is: %f" % mean_squared_error(y_train, model.predict(x_train)))
print("MSE for validation data is: %f" % mean_squared_error(y_test, model.predict(x_test)))
return accuracy
real_train_tar=np.expm1(train_linear_tar)
"""
. Lasso model
"""
lassocv = LassoCV(alphas = np.logspace(-5, 4, 400), )
lassocv.fit(train_linear_fea, train_linear_tar)
lassocv_score = lassocv.score(train_linear_fea, train_linear_tar)
lassocv_alpha = lassocv.alpha_
print("Best alpha : ", lassocv_alpha, "Score: ",lassocv_score)
start=time.time()
lasso =Lasso(normalize = True)
lasso.set_params(alpha=lassocv_alpha,max_iter = 10000)
lasso.fit(x_train, y_train)
end=time.time()
mean_squared_error(y_test, lasso.predict(x_test))
coef_lasso=pd.Series(lassocv.coef_, index=x_train.columns).sort_values(ascending =False)
evaluate(lasso,x_test,y_test,x_train,y_train)
print('Time elapsed: %.4f seconds' % (end-start))
y_lasso_predict=lasso.predict(train_linear_fea)
x_line = np.arange(700000)
y_line=x_line
plt.scatter(real_train_tar,np.expm1(y_lasso_predict))
plt.plot(x_line, y_line, color='r')
plt.xlabel('Actual Sale Price')
plt.ylabel('Predict Sle Price')
test_prediction_lasso=np.expm1(lasso.predict(test_linear))
"""
. Ridge model
"""
ridgecv = RidgeCV(alphas = np.logspace(-5, 4, 400))
ridgecv.fit(x_train, y_train)
ridgecv_score = ridgecv.score(x_train, y_train)
ridgecv_alpha = ridgecv.alpha_
print("Best alpha : ", ridgecv_alpha, "Score: ",ridgecv_score)
coef=pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending =False)
start=time.time()
ridge =Ridge(normalize = True)
ridge.set_params(alpha=ridgecv_alpha,max_iter = 10000)
ridge.fit(x_train, y_train)
end=time.time()
mean_squared_error(y_test, ridge.predict(x_test))
coef_ridge=pd.Series(ridgecv.coef_, index=x_train.columns).sort_values(ascending =False)
evaluate(ridge,x_test,y_test,x_train,y_train)
print('Time elapsed: %.4f seconds' % (end-start))
y_ridge_predict=ridge.predict(train_linear_fea)
x_line = np.arange(700000)
y_line=x_line
plt.scatter(real_train_tar,np.expm1(y_ridge_predict))
plt.plot(x_line, y_line, color='r')
plt.xlabel('Actual Sale Price')
plt.ylabel('Predict Sle Price')
test_prediction_ridge=np.expm1(ridge.predict(test_linear))
"""
. Random Forest
"""
#train=train.drop(columns=['DateSold'])
#test=test.drop(columns=['DateSold'])
#X_train=train.drop(columns=['SalePrice'])
#Y_train=train['SalePrice']
X_train=train_linear_fea
Y_train=train_linear_tar
x_train_rf, x_test_rf, y_train_rf, y_test_rf = train_test_split(X_train, Y_train,test_size=0.2, random_state=0)
n_estimators = [int(x) for x in np.linspace(start = 100, stop = 2000, num = 20)]
max_features = ['auto', 'sqrt']
max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
random_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
rf = RandomForestRegressor()
# Random search of parameters, using 3 fold cross validation,
# search across 100 different combinations, and use all available cores
#
rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)
rf_random.fit(X_train, Y_train)
#rf_random.fit(x_train_rf, y_train_rf)
rf_random.best_params_
#Random search allowed us to narrow down the range for each hyperparameter. Now that we know where to concentrate our search,
# we can explicitly specify every combination of settings to try.
param_grid = {
'bootstrap': [False],
'max_depth': [80, 90, 100, 110,120,130],
'max_features': [2, 3],
'min_samples_leaf': [1,2,3, 4],
'min_samples_split': [2,4,6,8, 10, 12],
'n_estimators': [600,700, 800, 900, 1000]
}
# Create a based model
rf = RandomForestRegressor()
# Instantiate the grid search model
grid_search = GridSearchCV(estimator = rf, param_grid = param_grid, cv = 3, n_jobs = -1, verbose = 2)
#grid_search.fit(x_train, y_train)
grid_search.fit(X_train, Y_train)
grid_search.best_params_
best_random = grid_search.best_estimator_
start=time.time()
best_random.fit(x_train_rf,y_train_rf)
end=time.time()
evaluate(best_random, x_test_rf, y_test_rf,x_train_rf,y_train_rf)
print('Time elapsed: %.4f seconds' % (end-start))
y_rf_predict=best_random.predict(train_linear_fea)
x_line = np.arange(700000)
y_line=x_line
plt.scatter(real_train_tar,np.expm1(y_rf_predict))
plt.plot(x_line, y_line, color='r')
plt.xlabel('Actual Sale Price')
plt.ylabel('Predict Sle Price')
importance_rf = pd.DataFrame({'features':train_linear_fea.columns, 'imp':best_random.feature_importances_}).\
sort_values('imp',ascending=False)
importance_top20_rf = importance_rf.iloc[:20,]
plt.barh(importance_top20_rf.features, importance_top20_rf.imp)
plt.xlabel('Feature Importance')
test_prediction_rf=np.expm1(best_random.predict(test_linear))
"""
. Xgboost
"""
learning_rate = [round(float(x), 2) for x in np.linspace(start = .1, stop = .2, num = 11)]
# Minimum for sum of weights for observations in a node
min_child_weight = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
# Maximum nodes in each tree
max_depth = [int(x) for x in np.linspace(1, 10, num = 10)]
n_estimators=[int(x) for x in np.linspace(start = 100, stop = 2000, num = 20)]
subsample=[0.3, 0.4,0.5,0.6, 0.7]
model = xgb.XGBRegressor()
random_grid = {'learning_rate': learning_rate,
'max_depth': max_depth,
'min_child_weight': min_child_weight,
'subsample': subsample,
'n_estimators':n_estimators
}
# Make a RandomizedSearchCV object with correct model and specified hyperparams
xgb_random = RandomizedSearchCV(estimator=model, param_distributions=random_grid, n_iter=1000, cv=5, verbose=2, random_state=42, n_jobs=-1)
start = time.time()
# Fit models
xgb_random.fit(X_train, Y_train)
xgb_random.best_params_
"""
best_params_={'learning_rate': 0.1,
'max_depth': 2,
'min_child_weight': 4,
'n_estimators': 900,
'subsample': 0.5}
"""
model_xgb = XGBRegressor(**xgb_random.best_params_)
#model_xgb = XGBRegressor(**best_params_)
start=time.time()
model_xgb.fit(x_train_rf,y_train_rf)
end=time.time()
evaluate(model_xgb, x_test_rf, y_test_rf,x_train_rf,y_train_rf)
print('Time elapsed: %.4f seconds' % (end-start))
y_xgb_predict=model_xgb.predict(train_linear_fea)
x_line = np.arange(700000)
y_line=x_line
plt.scatter(real_train_tar,np.expm1(y_xgb_predict))
plt.plot(x_line, y_line, color='r')
plt.xlabel('Actual Sale Price')
plt.ylabel('Predict Sle Price')
importance_xgb = pd.DataFrame({'features':train_linear_fea.columns, 'imp':model_xgb.feature_importances_}).\
sort_values('imp',ascending=False)
importance_top20_xgb = importance_xgb.iloc[:20,]
plt.barh(importance_top20_xgb.features, importance_top20_xgb.imp)
plt.xlabel('Feature Importance')
test_prediction_xgb=np.expm1(model_xgb.predict(test_linear))
return(test_prediction_lasso, test_prediction_ridge, test_prediction_rf, test_prediction_xgb,y_lasso_predict, y_ridge_predict, y_rf_predict, y_xgb_predict)
def return_a_reg(reg_name):
if reg_name == 'RR':
reg = Ridge()
reg.set_params(alpha=1)
return reg
if reg_name == 'KRR':
reg = KernelRidge()
reg.set_params(kernel='rbf', alpha=1)
return reg
if reg_name == 'SVR':
reg = SVR()
reg.set_params(kernel='rbf', C=10)
return reg
if reg_name == 'KNR':
reg = KNeighborsRegressor()
reg.set_params(n_neighbors=10)
return reg
if reg_name == 'GP':
reg = gaussian_process.GaussianProcess()
reg.set_params(regr='constant', corr='squared_exponential', theta0=1e-2, thetaL=1e-4, thetaU=1e-1)
return reg
if reg_name == 'DT':
reg = tree.DecisionTreeRegressor()
reg.set_params(max_depth=8, min_samples_leaf=5, random_state=0)
return reg
elif reg_name == 'RF':
reg = RandomForestRegressor()
reg.set_params(n_estimators=100, max_depth=5, min_samples_leaf=5, random_state=0)
return reg
elif reg_name == 'GBDT':
reg = GradientBoostingRegressor()
reg.set_params(n_estimators=100, max_depth=5, min_samples_leaf=5, random_state=0)
return reg
def set_params(self, C=None):
if self.l1:
Lasso.set_params(self, alpha=C)
else:
Ridge.set_params(self, alpha=1.0/(2.0*C))
return self
# nodebox section end
clf = Ridge()
X, y, w = make_regression(n_samples=10, n_features=10, coef=True,
random_state=1, bias=3.5)
coefs = []
errors = []
alphas = np.logspace(-6, 6, 200)
# Train the model with different regularisation strengths
for a in alphas:
clf.set_params(alpha=a)
clf.fit(X, y)
coefs.append(clf.coef_)
errors.append(mean_squared_error(clf.coef_, w))
# Display results
plt.figure(figsize=(20, 6))
plt.subplot(121)
ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Ridge coefficients as a function of the regularization')
plt.axis('tight')
clf = Ridge()
X, y, w = make_regression(n_samples=10,
n_features=10,
coef=True,
random_state=1,
bias=3.5)
coefs = []
errors = []
alphas = np.logspace(-6, 6, 200)
# Train the model with different regularisation strengths
for a in alphas:
clf.set_params(alpha=a)
clf.fit(X, y)
coefs.append(clf.coef_)
errors.append(mean_squared_error(clf.coef_, w))
# Display results
plt.figure(figsize=(20, 6))
plt.subplot(121)
ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Ridge coefficients as a function of the regularization')
plt.axis('tight')
X_test_sc = scaler_X.transform(X_test)
scaler_y = StandardScaler(with_mean=True, with_std=True)
y_train_sc = scaler_y.fit_transform(y_train.reshape(-1, 1)).ravel()
y_test_sc = scaler_y.transform(y_test.reshape(-1, 1)).ravel()
n_alphas = 50
alphas = np.logspace(-1, 8, n_alphas)
ridge = Ridge(fit_intercept=True)
kernel_ridge = KernelRidge(kernel='poly', gamma=1, degree=3, coef0=1)
test_scores_ridge = []
test_scores_kernel = []
for alpha in alphas:
ridge.set_params(alpha=alpha)
ridge.fit(X_train_sc, y_train_sc)
test_mse = mean_squared_error_scorer(ridge, X_test_sc, y_test_sc)
test_scores_ridge.append(test_mse)
kernel_ridge.set_params(alpha=alpha)
kernel_ridge.fit(X_train_sc, y_train_sc)
test_mse = mean_squared_error_scorer(kernel_ridge, X_test_sc, y_test_sc)
test_scores_kernel.append(test_mse)
poly = PolynomialNetworkRegressor(degree=3, n_components=2, tol=1e-3,
warm_start=True, random_state=0)
test_scores_poly = []
test = df[df.train_test__train ==0]
X_test = test.drop(['SalePrice','train_test__train'], axis=1)
alphas = 10**np.linspace(10,-2,100)*0.5
alphas.shape
ridge = Ridge(normalize = True)
coefs = []
for a in alphas:
ridge.set_params(alpha = a)
ridge.fit(X_train, y_train)
coefs.append(ridge.coef_)
np.shape(coefs)
ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
plt.axis('tight')
plt.xlabel('alpha')
plt.ylabel('weights')
N = X.shape[0]
X.insert(X.shape[1], 'intercept', np.ones(N))
y = df_scaled['lpsa']
#Pregunta A
X = X.drop('intercept', axis=1)
Xtrain = X[istrain]
ytrain = y[istrain]
names_regressors = [
"Lcavol", "Lweight", "Age", "Lpbh", "Svi", "Lcp", "Gleason", "Pgg45"
]
alphas_ = np.logspace(4, -1, base=10)
coefs = []
model = Ridge(fit_intercept=True, solver='svd')
for a in alphas_:
model.set_params(alpha=a)
model.fit(Xtrain, ytrain)
coefs.append(model.coef_)
ax = plt.gca()
for y_arr, label in zip(np.squeeze(coefs).T, names_regressors):
plt.plot(alphas_, y_arr, label=label)
plt.legend()
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1]) #reverse axis
plt.xlabel('Alpha')
plt.ylabel('Weights')
plt.title('Regularization Path RIDGE')
plt.axis('tight')
plt.legend(loc=2)
plt.show()
