import matplotlib.pyplot as plt
import numpy as np
from keras import Sequential
from keras.callbacks import EarlyStopping
from keras.layers import Dense, Dropout
from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler
if __name__ == '__main__':
X = np.load('X.npy')
Y = np.load('Y.npy')
clf = Ridge(alpha=1.0)
clf.fit(X, Y)
# scaler = StandardScaler()
# X = scaler.fit_transform(X)
#
# # define and fit the final model
# model = Sequential()
# model.add(Dense(10, input_dim=9, activation='relu'))
# model.add(Dropout(0.3))
# model.add(Dense(5, input_dim=9, activation='relu'))
# model.add(Dense(1, activation='linear'))
# model.compile(loss='mse', optimizer='adam')
#
# history = model.fit(X, Y, epochs=300, verbose=1, validation_split=0.05)
# # prediction = model.predict(X)
# # # show the inputs and predicted outputs
# model_json = model.to_json()
np.round(m9, 4)
# Piece-wise Linear Regression
PW_Illus = pd_read_csv("Data/PW_Illus.csv", delimiter=',')
X, Y = PW_Illus.X, PW_Illus.Y
PW_Illus['XL14'] = X * (X < 14)
PW_Illus['XGT14LT30'] = X * (X >= 14) * (X < 30)
PW_Illus['XGT30'] = X * (X >= 30)
PW_Final = smf.ols(formula='Y~XL14+XGT14LT30+XGT30', data=PW_Illus).fit()
print(PW_Final.params)
# Regularization linear model
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import Ridge
poly3 = PolynomialFeatures(degree=3)
X3 = poly3.fit_transform(X)
ridge = Ridge(fit_intercept=True, alpha=0.5)
ridge.fit(X3, Y)
# Logistic Regression and Regularization
from sklearn.linear_model import LogisticRegression
sat = pd.read_csv("Data/Sat.csv", delimiter=',')
y = np.array(sat[["Pass"]])
X = np.array(sat[["Sat"]])
model = LogisticRegression()
f1 = model.fit(X, y)
f1.intercept_, f1.coef_
model2 = LogisticRegression(penalty='l2', C=1e10)
f2 = model2.fit(X, y)
f2.intercept_, f2.coef_
def f(x):
""" function to approximate by polynomial interpolation"""
return x * np.sin(x)
# generate points used to plot
x_plot = np.linspace(0, 10, 100)
# generate points and keep a subset of them
x = np.linspace(0, 10, 100)
rng = np.random.RandomState(0)
rng.shuffle(x)
x = np.sort(x[:20])
y = f(x)
pl.plot(x_plot, f(x_plot), label="ground truth")
pl.scatter(x, y, label="training points")
for degree in [3, 4, 5]:
ridge = Ridge()
ridge.fit(np.vander(x, degree + 1), y)
pl.plot(x_plot,
ridge.predict(np.vander(x_plot, degree + 1)),
label="degree %d" % degree)
pl.legend(loc='lower left')
pl.show()
for i in range(0, 12):
x2_sc['x' + str(i)] = list([x2[j]**i for j in range(len(x2))])
x2_sc['y' + str(i)] = list([z2[j]**i for j in range(len(z2))])
x2_sc['x-' + str(i)] = list([x2[j]**-i for j in range(len(x2))])
x2_sc['y-' + str(i)] = list([z2[j]**-i for j in range(len(z2))])
X_train, X_test, y_train, y_test = train_test_split(x2_sc,
y2,
test_size=0.5,
random_state=10)
mse = list()
for i in range(1, 26):
rr = Ridge(alpha=i)
rr.fit(X_train, y_train)
y_pred_rr = rr.predict(X_test)
mse.append(mean_squared_error(y_test, y_pred_rr))
alph = np.argmin(mse) + 1
regr2 = Ridge(alpha=alph)
regr2.fit(x2_sc, y2)
y2_pred = regr2.predict(x2_sc)
fig2, ax2 = plt.subplots(1, 1)
ax2.set_ylabel("distance (Kilometers)")
ax2.set_xlabel("rtt (ms)")
ax2.set_title(
"rtt hops distance relation between planetlab landmarks \n(upmc_netmet slice nodes only)"
)
import numpy as np
from scipy.stats import uniform as sp_rand
from sklearn import datasets
from sklearn.linear_model import Ridge
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
dataset = datasets.load_diabetes()
print(dataset.data.shape)
print(dataset.target.shape)
model = Ridge()
model.fit(dataset.data, dataset.target)
print('Score with default parameters = ',
model.score(dataset.data, dataset.target))
alphas = np.array([1, 0.1, 0.01, 0.001, 0.0001, 0])
grid = GridSearchCV(estimator=model, param_grid=dict(alpha=alphas))
grid.fit(dataset.data, dataset.target)
print('Score with Grid Search parameters = ', grid.best_score_,
'best alpha = ', grid.best_estimator_.alpha)
param_grid = {'alpha': sp_rand()}
rand_grid_search = RandomizedSearchCV(estimator=model,
param_distributions=param_grid,
n_iter=100)
rand_grid_search.fit(dataset.data, dataset.target)
print('Score with Random Grid Search parameters = ',
# 线性回归模型
if False:
df_train = pd.get_dummies(df_train)
y = df_train['current_price']
X_train, X_test, y_train, y_test = train_test_split(df_train, y, test_size=0.3, random_state=0)
from sklearn.linear_model import Ridge, RidgeCV, ElasticNet, LassoCV, LassoLarsCV
from sklearn.model_selection import cross_val_score
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 5))
return(rmse)
model_ridge = Ridge()
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]
cv_ridge = pd.Series(cv_ridge, index = alphas)
cv_ridge.plot(title = "Validation - Just Do It")
plt.xlabel("alpha")
plt.ylabel("rmse")
plt.show()
print(cv_ridge.min())
model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005]).fit(X_train, y)
print(rmse_cv(model_lasso).mean())
coef = pd.Series(model_lasso.coef_, index = X_train.columns)
print("Lasso picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables")
imp_coef = pd.concat([coef.sort_values().head(10), coef.sort_values().tail(10)])
"""
LASSO MODEL
"""
# L1 regularization
lasso_linear = linear_model.Lasso(alpha=1.0)
lasso_linear.fit(x_train, y_train)
# evaluating L1 regularized model
score_lasso_trained = lasso_linear.score(x_test, y_test)
print "Lasso model scored:", score_lasso_trained
"""
RIDGE MODEL
"""
# L2 regularization
ridge_linear = Ridge(alpha=1.0)
ridge_linear.fit(x_train, y_train)
# evaluating L2 regularized model
score_ridge_trained = ridge_linear.score(x_test, y_test)
print "Ridge model scored:", score_ridge_trained
# saving model
joblib.dump(linear, "models/linear_model_v1.pkl")
# loading model
clf = joblib.load("models/linear_model_v1.pkl")
predicted = clf.predict(x_test)
print "Predicted test:", predicted
"""
X_test = X[nrow_train:]
del merge
del sparse_merge
del vectorizer
del tfidf_transformer
gc.collect()
X_train, X_test = intersect_drop_columns(X_train, X_test, min_df=1)
print(
f'[{time() - start_time}] Drop only in train or test cols: {X_train.shape[1]}'
)
gc.collect()
ridge = Ridge(solver='auto',
fit_intercept=True,
alpha=0.4,
max_iter=200,
normalize=False,
tol=0.01)
ridge.fit(X_train, y_train)
print(
f'[{time() - start_time}] Train Ridge completed. Iterations: {ridge.n_iter_}'
)
predsR = ridge.predict(X_test)
print(f'[{time() - start_time}] Predict Ridge completed.')
submission.loc[:, 'price'] = np.expm1(predsR)
submission.loc[submission['price'] < 0.0, 'price'] = 0.0
submission.to_csv("submission_ridge.csv", index=False)
def train_linear_model(
X,
y,
random_state=1,
test_size=0.2,
regularization_type="elasticnet",
k_fold=5,
max_iter=1000000,
tol=0.0001,
l1_ratio=None,
):
"""
Function to train linear model with regularization and cross-validation.
Args:
X (pandas.DataFrame): dataframe of descriptors.
y (pandas.DataFrame): dataframe of cycle lifetimes.
random_state (int): seed for train/test split.
test_size (float): proportion of the dataset reserved for model evaluation.
regularization_type (str): lasso or ridge or elastic-net (with cv).
k_fold (int): k in k-fold cross-validation.
max_iter (int): maximum number of iterations for model fitting.
tol (float): tolerance for optimization.
l1_ratio ([float]): list of lasso to ridge ratios for elasticnet.
Returns:
sklearn.linear_model.LinearModel: fitted model.
mu (float): Mean value of descriptors used in training.
s (float): Std dev of descriptors used in training.
"""
if l1_ratio is None:
l1_ratio = [0.1, 0.5, 0.7, 0.9, 0.95, 1]
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=test_size, random_state=random_state)
# Standardize (training) data after train/test split
mu = np.mean(X_train, axis=0)
s = np.std(X_train, axis=0)
X_scaled = (X_train - mu) / s
hyperparameters = {
"random_state": random_state,
"test_size": test_size,
"k_fold": k_fold,
"tol": tol,
"max_iter": max_iter,
}
if regularization_type == "lasso" and y.shape[1] == 1:
lassocv = LassoCV(fit_intercept=True,
alphas=None,
tol=tol,
cv=k_fold,
max_iter=max_iter)
lassocv.fit(X_scaled, y_train.values.ravel())
# Set optimal alpha and refit model
alpha_opt = lassocv.alpha_
linear_model = Lasso(fit_intercept=True,
alpha=alpha_opt,
max_iter=max_iter)
linear_model.fit(X_scaled, y_train.values)
hyperparameters["l1_ratio"] = 1
elif regularization_type == "ridge" and y.shape[1] == 1:
ridgecv = RidgeCV(fit_intercept=True, alphas=None, cv=k_fold)
ridgecv.fit(X_scaled, y_train.values.ravel())
# Set optimal alpha and refit model
alpha_opt = ridgecv.alpha_
linear_model = Ridge(fit_intercept=True, alpha=alpha_opt)
linear_model.fit(X_scaled, y_train)
hyperparameters["l1_ratio"] = 0
elif regularization_type == "elasticnet" and y.shape[1] == 1:
elasticnetcv = ElasticNetCV(
fit_intercept=True,
normalize=False,
alphas=None,
cv=k_fold,
l1_ratio=l1_ratio,
max_iter=max_iter,
)
elasticnetcv.fit(X_scaled, y_train.values.ravel())
# Set optimal alpha and l1_ratio. Refit model
alpha_opt = elasticnetcv.alpha_
l1_ratio_opt = elasticnetcv.l1_ratio_
linear_model = ElasticNet(
fit_intercept=True,
normalize=False,
l1_ratio=l1_ratio_opt,
alpha=alpha_opt,
max_iter=max_iter,
)
linear_model.fit(X_scaled, y_train)
hyperparameters["l1_ratio"] = l1_ratio_opt
# If more than 1 outcome present, perform multitask regression
elif regularization_type == "elasticnet" and y.shape[1] > 1:
multi_elasticnet_CV = MultiTaskElasticNetCV(
fit_intercept=True,
cv=k_fold,
normalize=False,
l1_ratio=l1_ratio,
max_iter=max_iter,
)
multi_elasticnet_CV.fit(X_scaled, y_train)
# Set optimal alpha and l1_ratio. Refit model
alpha_opt = multi_elasticnet_CV.alpha_
l1_ratio_opt = multi_elasticnet_CV.l1_ratio_
linear_model = MultiTaskElasticNet(fit_intercept=True,
normalize=False,
max_iter=max_iter)
linear_model.set_params(alpha=alpha_opt, l1_ratio=l1_ratio_opt)
linear_model.fit(X_scaled, y_train)
hyperparameters["l1_ratio"] = l1_ratio_opt
else:
raise NotImplementedError
y_pred = linear_model.predict((X_test - mu) / s)
Rsq = linear_model.score((X_test - mu) / s, y_test)
# Compute 95% confidence interval
# Multioutput = 'raw_values' provides prediction error per output
pred_actual_ratio = [x / y for x, y in zip(y_pred, np.array(y_test))]
relative_prediction_error = 1.96 * np.sqrt(
mean_squared_error(np.ones(y_pred.shape),
pred_actual_ratio,
multioutput="raw_values") / y_pred.shape[0])
hyperparameters["alpha"] = alpha_opt
return linear_model, mu, s, relative_prediction_error, Rsq, hyperparameters
def main(config_path, sigma_in, signal_length, alpha):
# hyper-parameter
with open(config_path, 'r') as f:
cfg = yaml.safe_load(f)
model_name = os.path.splitext(os.path.basename(config_path))[0]
os.makedirs('results/', exist_ok=True)
save_path = f'results/w_ridge/'
os.makedirs(save_path, exist_ok=True)
# モデルのロード
torch.manual_seed(1)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = RecurrentNeuralNetwork(n_in=1, n_out=2, n_hid=cfg['MODEL']['SIZE'], device=device,
alpha_time_scale=0.25, beta_time_scale=cfg['MODEL']['BETA'],
activation=cfg['MODEL']['ACTIVATION'],
sigma_neu=cfg['MODEL']['SIGMA_NEU'],
sigma_syn=cfg['MODEL']['SIGMA_SYN'],
use_bias=cfg['MODEL']['USE_BIAS'],
anti_hebbian=cfg['MODEL']['ANTI_HEBB']).to(device)
model_path = f'trained_model/romo/{model_name}/epoch_{cfg["TRAIN"]["NUM_EPOCH"]}.pth'
model.load_state_dict(torch.load(model_path, map_location=device))
model.eval()
sample_num = 5000
neural_dynamics = np.zeros((sample_num, 61, model.n_hid))
input_signal, omega_1_list, omega_2_list = romo_signal(sample_num, signal_length=signal_length,
sigma_in=sigma_in)
input_signal_split = np.split(input_signal, sample_num // cfg['TRAIN']['BATCHSIZE'])
# メモリを圧迫しないために推論はバッチサイズごとに分けて行う。
for i in range(sample_num // cfg['TRAIN']['BATCHSIZE']):
hidden = torch.zeros(cfg['TRAIN']['BATCHSIZE'], model.n_hid)
hidden = hidden.to(device)
inputs = torch.from_numpy(input_signal_split[i]).float()
inputs = inputs.to(device)
hidden_list, outputs, _, _ = model(inputs, hidden)
hidden_list_np = hidden_list.cpu().detach().numpy()
neural_dynamics[i * cfg['TRAIN']['BATCHSIZE']: (i + 1) * cfg['TRAIN']['BATCHSIZE']] = hidden_list_np
sample_X_1 = np.zeros([30 * sample_num, model.n_hid])
sample_X_2 = np.zeros([sample_num, model.n_hid])
sample_y_1 = np.zeros([30 * sample_num])
sample_y_2 = np.zeros(sample_num)
for i in range(sample_num):
sample_X_1[i * 30: (i + 1) * 30, :] = neural_dynamics[i, 15:45, :] ** 2
sample_X_2[i, :] = neural_dynamics[i, 55, :] ** 2
sample_y_1[i * 30: (i + 1) * 30] = omega_1_list[i]
sample_y_2[i] = omega_2_list[i]
# 訓練データとテストデータを分離
train_X_1, test_X_1, train_y_1, test_y_1 = train_test_split(sample_X_1, sample_y_1, random_state=0)
train_X_2, test_X_2, train_y_2, test_y_2 = train_test_split(sample_X_2, sample_y_2, random_state=0)
ridge_1 = Ridge(alpha=alpha)
ridge_1.fit(train_X_1, train_y_1)
ridge_2 = Ridge(alpha=alpha)
ridge_2.fit(train_X_2, train_y_2)
np.save(os.path.join(save_path, f'{model_name}_omega_1.npy'), ridge_1.coef_)
np.save(os.path.join(save_path, f'{model_name}_omega_2.npy'), ridge_2.coef_)
from sklearn.model_selection import train_test_split
from sklearn.linear_model import Lasso, Ridge
from sklearn.model_selection import GridSearchCV
if __name__ == "__main__":
#1. pandas读入数据
data = pd.read_csv('Advertising.csv') # TV、Radio、Newspaper、Sales
x = data[['TV', 'Radio', 'Newspaper']]
y = data['Sales']
print(x)
print(y)
#2. 将数据集分为训练集和测试集，默认将75%的数据划分到训练集中。
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=1)
# print x_train, y_train
model1 = Ridge()
model2 = Lasso()
#3. 设置α,并拟合模型
alpha_can = np.logspace(-3, 2, 10) #创建等比数列
#cv=5使用五折交叉验证，Rideg()需要一个α参数来设定正则化系数，默认是1.
ridge_model = GridSearchCV(model1, param_grid={'alpha': alpha_can}, cv=5)
ridge_model.fit(x, y)
lasso_model = GridSearchCV(model2, param_grid={'alpha': alpha_can}, cv=5)
lasso_model.fit(x, y)
#best_params_参数是最佳λ
print('Ridge回归模型的验证参数：\n', ridge_model.best_params_)
print('Lasso回归模型的验证参数：\n', lasso_model.best_params_)
#4. 查看预测结果的mse和rmse
y_hat1 = ridge_model.predict(np.array(x_test))
# }
# }
# model = make_pipeline(PolynomialFeatures(4), Ridge())
# model.fit(X, best_qsift_list)
# y_plot = model.predict(x_plot)
#
# best_qsift_trace = {
# "x": x_plot,
# "y": y_plot,
# "mode": "lines+markers",
# "name": "qSIFT q=0.6",
# "type": "scatter"
# }
model = make_pipeline(PolynomialFeatures(4), Ridge())
model.fit(X, surf_list)
y_plot = model.predict(x_plot)
surf_trace = {
"x": x_plot,
"y": y_plot,
"line": {
"shape": "spline"
},
"mode": "lines+markers",
"name": "SURF",
"type": "scatter",
"marker": {
"symbol": "triangle-down-open",
"size": 12
from sklearn.linear_model import Ridge
import matplotlib.pyplot as plt
import mglearn
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
X, y = mglearn.datasets.load_extended_boston()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
lr = LinearRegression().fit(X_train, y_train)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
ridge = Ridge().fit(X_train, y_train)
ridge10 = Ridge(alpha=10).fit(X_train, y_train)
ridge01 = Ridge(alpha=0.1).fit(X_train, y_train)
plt.plot(ridge10.coef_, '^', label="Ridge alpha=10")
plt.plot(ridge.coef_, 's', label="Ridge alpha=1")
plt.plot(ridge01.coef_, 'v', label="Ridge alpha=0.1")
plt.plot(lr.coef_, 'o', label="LinearRegression")
plt.xlabel("coefficient list")
plt.ylabel("coefficient size")
xlims = plt.xlim()
plt.hlines(0, xlims[0], xlims[1])
plt.xlim(xlims)
plt.ylim(-25, 25)
plt.legend()
plt.show()
path = os.path.join(data_path,'alpha_dict.npz')
data = np.load(path)
data_dict = data['a']
data_dict = data_dict.reshape(1)
data_dict = data_dict[0]
alpha = data_dict[layer][subject]
create_saving_folder(main_path)
filename_stimuli = "/home/brain/datasets/SherlockMerlin_ds001110/stimuli/Soundnet_features/sherlock_layer_" + str(layer) + ".npy"
filename_mask = "/home/brain/datasets/SherlockMerlin_ds001110/sub-" + str(subject) + "/func/sub-" + str(subject) + "_task-SherlockMovie_bold_space-MNI152NLin2009cAsym_brainmask.nii.gz"
filename_irm = "/home/brain/datasets/SherlockMerlin_ds001110/sub-" + str(subject) + "/func/sub-" + str(subject) + "_task-SherlockMovie_bold_space-MNI152NLin2009cAsym_preproc.nii.gz"
print('\nData loaded successfully')
estimator = Ridge(alpha)
X_train,X_test,y_train,y_test,masker,meanepi = init(subject,layer,filename_irm,filename_mask,filename_stimuli)
data_path_vector = os.path.join(data_path,'vector_nilearn.npz')
data_path_meanepi = os.path.join(data_path,'meanepi.nii.gz')
np.savez_compressed(data_path_vector,a=X_train,b=X_test,c=y_train,d=y_test)
meanepi.to_filename(data_path_meanepi)
print('Data saved successfully')
print('X shape: ',X_test.shape)
print('Y shape: ',y_train.shape,'\n')
scores = reference_model_kmeans(X_train,X_test,y_train,y_test,estimator)
score_img = masker.inverse_transform(scores)
for num_clust in range(2,11):
from sklearn.pipeline import make_pipeline
file = open("data.txt", "r")
data = file.read().split("\n")
data_x = []
data_y = []
for i in range(len(data)):
data_x.append(i)
data_y.append(float(data[i]))
x = np.array(data_x).reshape((-1, 1))
y = np.array(data_y)
plt.scatter(x, y, color="blue")
poly = make_pipeline(PolynomialFeatures(degree=3), Ridge())
poly.fit(x, y)
y_poly = poly.predict(x)
plt.plot(x, y_poly, color="red")
pred_x = np.array([len(x)]).reshape((-1, 1))
pred_y = poly.predict(pred_x)
plt.scatter(pred_x, pred_y, color="red")
plt.show()
linear_val_mae = mean_absolute_error(inv_y(linear_val_predictions), inv_y(val_y))
mae_compare['LinearRegression'] = linear_val_mae
# print("Validation MAE for Linear Regression Model: {:,.0f}".format(linear_val_mae))
# Lasso ==============================================================
lasso_model = Lasso(alpha=0.0005, random_state=5)
lasso_model.fit(train_X, train_y)
lasso_val_predictions = lasso_model.predict(val_X)
lasso_val_mae = mean_absolute_error(inv_y(lasso_val_predictions), inv_y(val_y))
mae_compare['Lasso'] = lasso_val_mae
# print("Validation MAE for Lasso Model: {:,.0f}".format(lasso_val_mae))
# Ridge ===============================================================
ridge_model = Ridge(alpha=0.002, random_state=5)
ridge_model.fit(train_X, train_y)
ridge_val_predictions = ridge_model.predict(val_X)
ridge_val_mae = mean_absolute_error(inv_y(ridge_val_predictions), inv_y(val_y))
mae_compare['Ridge'] = ridge_val_mae
# print("Validation MAE for Ridge Regression Model: {:,.0f}".format(ridge_val_mae))
# ElasticNet ===========================================================
elastic_net_model = ElasticNet(alpha=0.02, random_state=5, l1_ratio=0.7)
elastic_net_model.fit(train_X, train_y)
elastic_net_val_predictions = elastic_net_model.predict(val_X)
elastic_net_val_mae = mean_absolute_error(inv_y(elastic_net_val_predictions), inv_y(val_y))
mae_compare['ElasticNet'] = elastic_net_val_mae
# print("Validation MAE for Elastic Net Model: {:,.0f}".format(elastic_net_val_mae))
best_ls = ls_score.rename(
columns={
'mean_test_score': 'Mean Test Score',
'mean_train_score': 'Mean Train Score'
}).iloc[ls_score['mean_test_score'].idxmax()]
# RLS algorithm
rls_params = {
'alpha': logspace(rls_min_lambda, rls_max_lambda,
rls_n_lambda_to_try)
}
print('%.2f - %s - %d - Training RLS...' %
(time.time() - big_ben, datetime.now().strftime("%H:%M:%S"),
attempt))
# solver='auto' different methods for the computational routines based on the data types
rls = Estimator.train_estimator(Ridge(solver='auto'),
xtr,
ytr,
rls_params,
folds=folds)
print('%.2f - %s - %d - Testing RLS...' %
(time.time() - big_ben, datetime.now().strftime("%H:%M:%S"),
attempt))
rls_score, rls_score_2 = Estimator.test_estimator(rls, xts, yts)
if saveResults:
rls_test_results.append([
rls_score_2, rls_score.iloc[
rls_score['mean_test_score'].idxmax()]['mean_test_score'],
rls_score.iloc[
rls_score['mean_test_score'].idxmax()]['mean_train_score']
])
# LinearRegression 클래스는 학습 데이터에 최적화되도록
# 학습을 하기때문에 테스트 데이터에 대한 일반화 성능이 감소됩니다.
# 이러한 경우 모든 특정 데이터를 적절히 활용할 수 있도록
# L2 제약 조건을 사용할 수 있으며, L2 제약조건으로 인하여
# 모델의 일반화 성능(테스트 데이터의 성능)이 증가하게 됩니다.
from sklearn.linear_model import Ridge
# Ridge 클래스의 하이퍼 파라메터 alpha
# alpha의 값이 커질수록 제약을 크게 설정
# (alpha의 값이 커질수돌 모든 특성들의 가중치의 값은
# 0 주변으로 위치함)
# alpha의 값이 작아질수록 제약이 약해짐
# (alpha의 값이 작아질수록 모든 특성들의 가중치의 값은
# 0에서 멀어짐)
# alpha의 값이 작아질수록 LinearRegression 클래스와 동일해짐
ridge_model = Ridge(alpha=100).fit(X_data, y_data)
print('Ridge평가(R2) : ', ridge_model.score(X_data, y_data))
import numpy as np
from matplotlib import pyplot as plt
# X데이터의 특성(컬럼) 개수 확인
print(X_data.shape[1])
# LinearRegression 객체의 가중치 변수 확인
print(lr_model.coef_)
# X축의 데이터
coef_range = np.arange(1, lr_model.coef_.shape[0] + 1)
plt.plot(coef_range, lr_model.coef_, 'ro')
plt.plot(coef_range, ridge_model.coef_, 'b^')
# Ridge
solver_settings = ('auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag',
'saga')
alpha_settings = np.logspace(-5, 5, 11)
for norm in lr_settings:
for solv in solver_settings:
for alph in alpha_settings:
# create and fit the model
rdg = Ridge(alpha=alph,
fit_intercept=True,
normalize=norm,
solver=solv).fit(X_train, y_train)
# record training set accuracy
train_score = rdg.score(X_train, y_train)
training_accuracy.append([train_score, "Ridge", norm, solv, alph])
# record generalization accuracy
test_score = rdg.score(X_test, y_test)
test_accuracy.append([test_score, "Ridge", norm, solv, alph])
# printing results
print("for {}, with norm {}, solv {}, alph {}".format(
"Ridge", norm, solv, alph))
print("train score: {:.2f}, test score: {:.2f}\n".format(
train_score, test_score))
Xm = Xtrain.as_matrix()
ym = ytrain.as_matrix()
from sklearn.linear_model import Ridge
import matplotlib.pylab as plt
#X = X.drop('intercept', axis=1)
#Xtrain = X[istrain]
#ytrain = y[istrain]
names_regressors = [
"Lcavol", "Lweight", "Age", "Lbph", "Svi", "Lcp", "Gleason", "Pgg45"
]
alphas_ = np.logspace(4, -1, base=10)
coefs = []
model = Ridge(fit_intercept=True, solver='svd')
for a in alphas_:
model.set_params(alpha=a)
model.fit(Xtrain, ytrain)
coefs.append(model.coef_)
ax = plt.gca()
for y_arr, label in zip(np.squeeze(coefs).T, names_regressors):
print alphas_.shape
print y_arr.shape
plt.plot(alphas_, y_arr, label=label)
plt.legend()
ax.set_xscale('log')
ax.set_xlim(ax.get_xlim()[::-1]) # reverse axis
plt.xlabel('alpha')
plt.ylabel('weights')
plt.title('Regularization Path RIDGE')
X_selected = X_scaled_std
# Split data into train and test without shuffling - test size: 20%
X_train, X_test, y_train, y_test, y_train_log, y_test_log = train_test_split(X_selected, y, y_log,
train_size=0.8, test_size=0.2,
shuffle=False)
print("Performing PCA")
# Perform PCA
pca = PCA(n_components=200)
X_pca = pca.fit_transform(X_scaled_std)
X_train_pca, X_test_pca = train_test_split(X_pca, train_size=0.8, test_size=0.2, shuffle=False)
print("Getting estimators")
# Get simple linear regression objects
ridge_reg = Ridge(normalize=False)
ridge_reg_cv = RidgeCV(alphas=(50.0, 100.0, 200.0), normalize=False)
lasso_reg = Lasso(normalize=False)
lasso_reg_cv = LassoCV(normalize=False, n_alphas=10)
reg_list = [("Ridge", ridge_reg),
("RidgeCV", ridge_reg_cv),
("Lasso", lasso_reg),
("LassoCV", lasso_reg_cv)]
print("Feature selection - RFE")
# Feature selection - done with recursive feature elimination
feature_selection_rfe = RFECV(ridge_reg, step=100, verbose=0, cv=ShuffleSplit(n_splits=5, train_size=0.8, test_size=0.2,
random_state=2973))
feature_selection_rfe.fit(X_train, y_train)
# Get new data set containing only selected features
X_train, X_test, y_train, y_test = data_split
# Scale both the features (X) and the target (y) to zero mean, unit variance
# (This is not necessary but makes the plots clearer)
scaler_X = StandardScaler(with_mean=True, with_std=True)
X_train_sc = scaler_X.fit_transform(X_train)
X_test_sc = scaler_X.transform(X_test)
scaler_y = StandardScaler(with_mean=True, with_std=True)
y_train_sc = scaler_y.fit_transform(y_train.reshape(-1, 1)).ravel()
y_test_sc = scaler_y.transform(y_test.reshape(-1, 1)).ravel()
n_alphas = 50
alphas = np.logspace(-1, 8, n_alphas)
ridge = Ridge(fit_intercept=True)
kernel_ridge = KernelRidge(kernel='poly', gamma=1, degree=3, coef0=1)
test_scores_ridge = []
test_scores_kernel = []
for alpha in alphas:
ridge.set_params(alpha=alpha)
ridge.fit(X_train_sc, y_train_sc)
test_mse = mean_squared_error_scorer(ridge, X_test_sc, y_test_sc)
test_scores_ridge.append(test_mse)
kernel_ridge.set_params(alpha=alpha)
kernel_ridge.fit(X_train_sc, y_train_sc)
test_mse = mean_squared_error_scorer(kernel_ridge, X_test_sc, y_test_sc)
test_scores_kernel.append(test_mse)
#data = data[cate_columns]
scaler1 = StandardScaler().fit(data)
scaler2 = MinMaxScaler().fit(data)
train_X = scaler2.transform(data[:train.shape[0]])
test = scaler2.transform(data[train.shape[0]:])
train_Y = target.values
train_X, test_X, train_Y, test_Y = train_test_split(train_X,
train_Y,
test_size=0.1,
random_state=1)
##############################--Ridge--########################################
ridge = Ridge(alpha=0.01, normalize=True, max_iter=1000, random_state=2019)
###############################--RFR--##########################################
myRFR = RandomForestRegressor(n_estimators=2000,
max_depth=10,
min_samples_leaf=10,
min_samples_split=0.001,
max_features='auto',
max_leaf_nodes=30,
min_weight_fraction_leaf=0.001,
random_state=10)
stack = myRFR
stack.fit(train_X, train_Y)
Y_pred = stack.predict(test_X)
print(mean_squared_error(test_Y, Y_pred))
r1_zscore[0] / d1_zscore[0] # In[75]: pop = r1_zscore / d1_zscore pop.mean() # In[76]: get_ipython().run_cell_magic(u'HTML', u'', u'<h1>Ridge Regression</h1>') # In[77]: from sklearn.linear_model import Ridge X_train_ridge, X_test_ridge, y_train_ridge, y_test_ridge = X_late_train, X_late_test, Y_late_train, Y_late_test clf = Ridge(alpha=1.0) clf.fit(X_train_ridge, y_train_ridge) # In[78]: y_res_ridgel = clf.predict(X_test_ridge) print y_res_ridgel # In[79]: from sklearn.linear_model import Ridge X_train_ridge, X_test_ridge, y_train_ridge, y_test_ridge = X_Carr_train, X_Carr_test, Y_Carr_train, Y_Carr_test clf = Ridge(alpha=1.0) clf.fit(X_train_ridge, y_train_ridge) # In[80]:
("lin_reg", LinearRegression()),
])
plot_learning_curves(polynomial_regression, X, y)
plt.axis([0, 80, 0, 3]) # not shown
#save_fig("learning_curves_plot") # not shown
plt.show() # not shown
# regularization!
# NB: should only regularize during trainig
# NB: scale the data (StandardScaler() ) before regularizing
# look at forluas for immediate understanding
# Ridge regression: keep weights small large alpha-> highly regularized
from sklearn.linear_model import Ridge
ridge_reg = Ridge(alpha=1, solver="cholesky", random_state=42)
ridge_reg.fit(X, y)
sgd_reg = SGDRegressor(max_iter=5, penalty="l2", random_state=42)
sgd_reg.fit(X, y.ravel())
# specifying l2 in sgdr means doing ridge regularization
# Lasso regression - like ridge, but uses l1 norm rather than l2
# eliminates weights of least important features, ie, does automatic feature selection
from sklearn.linear_model import Lasso
lasso_reg = Lasso(alpha=0.1)
lasso_reg.fit(X, y)
# elastic net - does a bit of both lasso and ridge, depending on settings
from sklearn.linear_model import ElasticNet
elastic_net = ElasticNet(alpha=0.1, l1_ratio=0.5, random_state=42)
res = get_sum_grouped_by_date(sales, 'years', 'Revenue', 'Order Date',
'MM/dd/yyyy', 'Total Revenue')
revenues = res.select("Revenue").rdd.map(lambda x: x[0]).collect()
years = res.select("years").rdd.map(lambda x: x[0]).collect()
y_train = revenues
start = min(years)
end = max(years)
x_plot = np.linspace(start, end, (end - start) + 1)
plt.scatter(x_plot,
y_train,
color='navy',
s=30,
marker='o',
label="training points")
colors = ['teal', 'yellowgreen', 'gold', 'red', 'green', 'violet', 'grey']
x_plot = x_plot.reshape(-1, 1)
plt.plot(years, revenues)
for count, degree in enumerate([2, 3]):
model = make_pipeline(PolynomialFeatures(degree), Ridge())
model.fit(x_plot, y_train)
y_plot = model.predict(x_plot)
plt.plot(x_plot,
y_plot,
color=colors[count],
linewidth=2,
label="degree %d" % degree)
plt.legend(loc='upper right')
plt.show()
dataset = read_csv("housingdata.csv", names=names)
X = dataset.iloc[:, 0:13].values
y = dataset.iloc[:, 13].values
kfold = KFold(n_splits=10, random_state=7)
scoring = 'neg_mean_squared_error'
# ALL LINEAR REGRESSION MODELS
# Linear Regression Model
model1 = LinearRegression()
results1 = cross_val_score(model, X, y, cv=kfold, scoring=scoring)
print(results.mean())
# Ridge Regression Model
model2 = Ridge()
results2 = cross_val_score(model2, X, y, cv=kfold, scoring=scoring)
print(results2.mean())
# LASSO Regression Model
model3 = Lasso()
results3 = cross_val_score(model3, X, y, scoring=scoring, cv=kfold)
print(results3.mean())
# ElasticNet Regression Model
model4 = ElasticNet()
results4 = cross_val_score(model4, X, y, scoring=scoring, cv=kfold)
print(results4.mean())
# ALL NON-LINEAR REGRESSION MODELS
strip_accents='unicode',
analyzer='char',
ngram_range=(1, 5),
max_features=30000)
char_vectorizer.fit(all_text)
train_char_features = char_vectorizer.transform(train_text)
test_char_features = char_vectorizer.transform(test_text)
train_features = hstack([train_char_features, train_word_features])
test_features = hstack([test_char_features, test_word_features])
losses = []
predictions = {'id': test['id']}
for class_name in class_names:
train_target = train[class_name]
classifier = Ridge(alpha=1.0, solver='sag')
cv_loss = np.mean(
cross_val_score(classifier,
train_features,
train_target,
cv=3,
scoring='roc_auc'))
losses.append(cv_loss)
print('CV score for class {} is {}'.format(class_name, cv_loss))
classifier.fit(train_features, train_target)
predictions[class_name] = classifier.predict(test_features)
print('Total CV score is {}'.format(np.mean(losses)))
# Create a boxplot of life expectancy per region
df.boxplot('life', 'Region', rot=60)
# Show the plot
plt.show()
# Create dummy variables: df_region
df_region = pd.get_dummies(df)
# Print the columns of df_region
print(df_region.columns)
# Create dummy variables with drop_first=True: df_region
df_region = pd.get_dummies(df, drop_first=True)
# Print the new columns of df_region
print(df_region.columns)
# Import necessary modules
from sklearn.linear_model import Ridge
from sklearn.model_selection import cross_val_score
# Instantiate a ridge regressor: ridge
ridge = Ridge(alpha=0.5, normalize=True)
# Perform 5-fold cross-validation: ridge_cv
ridge_cv = cross_val_score(ridge, X, y, cv=5)
# Print the cross-validated scores
print(ridge_cv)
model_Forest.fit(X, y)
y_pred = model_Forest.predict(X_pred)
plot_pred(y, y_pred, "Random Forest")
model_xgb = xgb.XGBRegressor()
model_xgb.fit(X, y)
y_pred = model_xgb.predict(X_pred)
plot_pred(y, y_pred, "xgboost")
model_linear = LinearRegression()
model_linear.fit(X, y)
y_pred = model_linear.predict(X_pred)
plot_pred(y, y_pred, "Linear regression")
model_ridge = Ridge()
model_ridge.fit(X, y)
y_pred = model_ridge.predict(X_pred)
plot_pred(y, y_pred, "Ridge regression")
class ensemble_model:
def __init__(self):
self.models = [
RandomForestRegressor(),
xgb.XGBRegressor(),
LinearRegression(),
Ridge()
]
