def parameter_select(X, y):
print X.shape, y.shape
##############################################################################
# LassoLarsIC: least angle regression with BIC/AIC criterion
# model_bic = LassoLarsIC(criterion='bic')
# model_bic.fit(X, y)
# alpha_bic_ = model_bic.alpha_
model_aic = LassoLarsIC(criterion='aic', max_iter=100000000)
model_aic.fit(X, y)
alpha_aic_ = model_aic.alpha_
print alpha_aic_
def plot_ic_criterion(model, name, color):
alpha_ = model.alpha_
alphas_ = model.alphas_
criterion_ = model.criterion_
plt.plot(-np.log10(alphas_), criterion_, '--', color=color,
linewidth=3, label='%s criterion' % name)
plt.axvline(-np.log10(alpha_), color=color, linewidth=3,
label='alpha: %s estimate' % name)
plt.xlabel('-log(alpha)')
plt.ylabel('criterion')
plt.figure()
plot_ic_criterion(model_aic, 'AIC', 'b')
# plot_ic_criterion(model_bic, 'BIC', 'r')
plt.legend()
plt.title('Information-criterion for model selection')
plt.show()
fields = iot.read_fields()
for i in xrange(len(fields)):
print str(fields[i]) +'\t'+ str(model_aic.coef_[i])
def test_lasso_lars_fit_copyX_behaviour(copy_X):
"""
Test that user input to .fit for copy_X overrides default __init__ value
"""
lasso_lars = LassoLarsIC(precompute=False)
rng = np.random.RandomState(0)
X = rng.normal(0, 1, (100, 5))
X_copy = X.copy()
y = X[:, 2]
lasso_lars.fit(X, y, copy_X=copy_X)
assert copy_X == np.array_equal(X, X_copy)
def lassoM(X, y, method):
if (method == "aic") or (method == "bic"):
lasso = LassoLarsIC(criterion=method, normalize=True)
lasso.fit(X, y)
elif method == "LassoCV":
lasso = LassoCV(cv=20).fit(X, y)
elif method == "LassoLarsCV":
lasso = LassoLarsCV(cv=20).fit(X, y)
#print(lasso.alpha_)
return lasso
def boot_coef(X, Y):
"""
Takes original data and new sampling index.
Returns coefficient vector.
"""
np.random.seed()
N = X.shape[0]
ind = np.random.choice(N, N) #resample step
clf = LassoLarsIC(criterion = 'aic')
clf.fit(X[ind,],Y[ind])
point_estimate = clf.coef_
return point_estimate
def test_lasso_lars_copyX_behaviour(copy_X):
"""
Test that user input regarding copy_X is not being overridden (it was until
at least version 0.21)
"""
lasso_lars = LassoLarsIC(copy_X=copy_X, precompute=False)
rng = np.random.RandomState(0)
X = rng.normal(0, 1, (100, 5))
X_copy = X.copy()
y = X[:, 2]
lasso_lars.fit(X, y)
assert copy_X == np.array_equal(X, X_copy)
class LassoLarsICImpl:
def __init__(self, **hyperparams):
self._hyperparams = hyperparams
self._wrapped_model = Op(**self._hyperparams)
def fit(self, X, y=None):
if y is not None:
self._wrapped_model.fit(X, y)
else:
self._wrapped_model.fit(X)
return self
def predict(self, X):
return self._wrapped_model.predict(X)
def _pruning(self, X, ee, order, causal_order):
""""""
n_features = X.shape[1]
# join X(t), X(t-1) and e(t-1)
X_joined = np.zeros(
(X.shape[0], X.shape[1] * (1 + order[0] + order[1])))
for p in range(1 + order[0]):
pos = n_features * p
X_joined[:, pos:pos + n_features] = np.roll(X[:, 0:n_features],
p,
axis=0)
for q in range(order[1]):
pos = n_features * (1 + order[0]) + n_features * q
X_joined[:, pos:pos + n_features] = np.roll(ee[:, 0:n_features],
q + 1,
axis=0)
# pruned by adaptive lasso
psi_omega = np.zeros(
(n_features, n_features * (1 + order[0] + order[1])))
for i, target in enumerate(causal_order):
predictors = [
j for j in range(X_joined.shape[1])
if j not in causal_order[i:]
]
# adaptive lasso
gamma = 1.0
lr = LinearRegression()
lr.fit(X_joined[:, predictors], X_joined[:, target])
weight = np.power(np.abs(lr.coef_), gamma)
reg = LassoLarsIC(criterion='bic')
reg.fit(X_joined[:, predictors] * weight, X_joined[:, target])
psi_omega[target, predictors] = reg.coef_ * weight
# split psi and omega
psis = np.zeros(((1 + order[0]), n_features, n_features))
for p in range(1 + order[0]):
pos = n_features * p
psis[p] = psi_omega[:, pos:pos + n_features]
omegas = np.zeros((order[1], n_features, n_features))
for q in range(order[1]):
pos = n_features * (1 + order[0]) + n_features * q
omegas[q] = psi_omega[:, pos:pos + n_features]
return psis, omegas
def __lasso_selected(data,data_test, response):
X = data.drop([response],axis=1).as_matrix()
y = np.array(data[response].tolist()).reshape((len(data),1))
#X = sm.add_constant(X)
#model = sm.OLS(y,X)
#m = model.fit_regularized(refit=True)
#yp = m.predict(data_test)
reg = LassoLarsIC(criterion='bic')
print y.shape,X.shape
reg.fit(X,y)
x = data_test.drop([response],axis=1).as_matrix().reshape((len(data_test),len(data_test.keys())-1))
yp = reg.predict(x)
te = np.mean((yp-np.array(data_test[response].tolist()))**2)
print reg.coef_,te
return
class r07546035_ICRegression(regression):
def trainAlgo(self):
self.model = LassoLarsIC(criterion=self.param['criterion'],
fit_intercept=self.param['fit_intercept'],
normalize=self.param['normalize'],
max_iter=self.param['max_iter'],
eps=self.param['eps'],
positive=self.param['positive'])
self.model.fit(self.inputData['X'], self.outputData['Y'])
def predictAlgo(self):
self.result['Y'] = self.model.predict(self.inputData['X'])
def mdl_1d(x, y):
"""builds univariate model to calculate AUC"""
lr = LogisticRegressionCV(scoring='roc_auc')
lars = LassoLarsIC(criterion='aic')
if x.nunique() > 10 and com.is_numeric_dtype(x):
x2 = sb_cutz(x)
series = pd.get_dummies(x2, dummy_na=True)
else:
series = pd.get_dummies(x, dummy_na=True)
lr.fit(series, y)
lars.fit(series, y)
try:
preds = (lr.predict_proba(series)[:, -1])
#preds = (preds > preds.mean()).astype(int)
except ValueError:
Tracer()()
# try:
# cm = confusion_matrix(y, (preds > y.mean()).astype(int))
# except ValueError:
# Tracer()()
aucz = roc_auc_score(y, preds)
ns = num_bin_stats(x, y)
nplot = plot_num(ns)
#plot = plot_confusion_matrix(cm, y)
imgdata = BytesIO()
nplot.savefig(imgdata)
imgdata.seek(0)
nplot = 'data:image/png;base64,' + \
quote(base64.b64encode(imgdata.getvalue()))
plt.close()
bplot = plot_bubble(ns)
imgdatab = BytesIO()
bplot.savefig(imgdatab)
imgdatab.seek(0)
bplot = 'data:image/png;base64,' + \
quote(base64.b64encode(imgdatab.getvalue()))
plt.close()
return aucz, nplot, bplot
class HistogramClassifier:
def __init__(self):
X, y = make_dataframe(letter_list)
self.columns = list(X.columns)
self.classifier = LassoLarsIC()
self.classifier.fit(X, y)
def predict(self, X):
counter = snippet_to_histogram(X, letter_list)
df = pd.DataFrame(columns=self.columns)
df = df.append(counter, ignore_index=True).fillna(0)
y = np.zeros(len(X))
for i in range(len(X)):
y[i] = self.classifier.predict(df)
y = round(y.sum() / len(X))
return y
def trained_model(self, data, dependent_var):
"""
Attempts to find the best lasso alpha value fitting a dataset using the BIC
metric: https://stats.stackexchange.com/questions/126898/tuning-alpha-parameter-in-lasso-linear-model-in-scikitlearn
"""
predictors = data.drop([dependent_var], axis=1)
# def bic_score(predictors, y, model):
# sse = sum((model.predict(predictors) - y.values[0])**2)
# s = np.count_nonzero(model.coef_)
# n = len(predictors.columns)
# cn = math.sqrt(n)/(s*s)
# print(math.log(sse/n) + s*math.log(n)/n*cn)
# return math.log(sse/n) + abs(s)*math.log(n)/n*cn
model = LassoLarsIC(criterion='bic')
model.fit(predictors, data[[dependent_var]])
return model
def run_lasso_lars_ic(X_train, y_train, X_test, y_test):
"""
ic: information criterion (AIC/BIC)
usually faster, but breaks up when sample size << feature size
:param X_train:
:param y_train:
:param X_test:
:param y_test:
:return:
"""
model_bic = LassoLarsIC(criterion="bic")
model_aic = LassoLarsIC(criterion="aic")
model_bic.fit(X_train, y_train)
model_aic.fit(X_train, y_train)
plot_ic_criterion(model_bic, 'BIC', 'b')
plot_ic_criterion(model_aic, 'AIC', 'r')
def aic(df_data):
X_data_pd = df_data.toPandas() # spark DF to pd_df
X1 = X_data_pd.values
y1 = X_data_pd['label'].values * 10000
rng = np.random.RandomState(42)
X = np.c_[X, rng.randn(X.shape[0], 14)] # add some bad features
X1 = np.c_[X1, rng.randn(X1.shape[0], 14)]
# normalize data as done by Lars to allow for comparison
X /= np.sqrt(np.sum(X**2, axis=0))
X1 /= np.sqrt(np.sum(X1**2, axis=0))
# #############################################################################
# LassoLarsIC: least angle regression with BIC/AIC criterion
model_aic = LassoLarsIC(criterion='aic')
t3 = time.time()
model_aic.fit(X1, y1)
t_aic = time.time() - t3
alpha_aic_ = model_aic.alpha_
def plot_ic_criterion(plot_model, name, color):
alpha_ = plot_model.alpha_
alphas_ = plot_model.alphas_
criterion_ = plot_model.criterion_
plt.plot(-np.log10(alphas_),
criterion_,
'--',
color=color,
linewidth=3,
label='%s criterion' % name)
plt.axvline(-np.log10(alpha_),
color=color,
linewidth=3,
label='alpha: %s estimate' % name)
plt.xlabel('-log(alpha)')
plt.ylabel('criterion')
plt.figure()
plot_ic_criterion(model_aic, 'AIC', 'b')
plt.legend()
plt.title(
'Information-criterion for model selection (training time %.3fs)' %
t_aic)
plt.show()
def _lassolarsic(*,
train,
test,
x_predict=None,
metrics,
criterion='aic',
fit_intercept=True,
verbose=False,
normalize=True,
precompute='auto',
max_iter=500,
eps=2.220446049250313e-16,
copy_X=True,
positive=False):
"""For more info visit :
https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LassoLarsIC.html#sklearn.linear_model.LassoLarsIC
"""
model = LassoLarsIC(criterion=criterion,
fit_intercept=fit_intercept,
verbose=verbose,
normalize=normalize,
precompute=precompute,
max_iter=max_iter,
eps=eps,
copy_X=copy_X,
positive=positive)
model.fit(train[0], train[1])
model_name = 'LassoLarsIC'
y_hat = model.predict(test[0])
if metrics == 'mse':
accuracy = _mse(test[1], y_hat)
if metrics == 'rmse':
accuracy = _rmse(test[1], y_hat)
if metrics == 'mae':
accuracy = _mae(test[1], y_hat)
if x_predict is None:
return (model_name, accuracy, None)
y_predict = model.predict(x_predict)
return (model_name, accuracy, y_predict)
def _pruning(self, X, B_taus, causal_order):
"""Prune edges"""
n_features = X.shape[1]
stacked = [np.flip(X, axis=0)]
for i in range(self._lags):
stacked.append(np.roll(stacked[-1], -1, axis=0))
blocks = np.array(list(zip(*stacked)))[:-self._lags]
for i in range(n_features):
causal_order_no = causal_order.index(i)
ancestor_indexes = causal_order[:causal_order_no]
obj = np.zeros((len(blocks)))
exp = np.zeros(
(len(blocks), causal_order_no + n_features * self._lags))
for j, block in enumerate(blocks):
obj[j] = block[0][i]
exp[j:] = np.concatenate(
[
block[0][ancestor_indexes].flatten(),
block[1:][:].flatten()
],
axis=0,
)
# adaptive lasso
gamma = 1.0
lr = LinearRegression()
lr.fit(exp, obj)
weight = np.power(np.abs(lr.coef_), gamma)
reg = LassoLarsIC(criterion="bic")
reg.fit(exp * weight, obj)
coef = reg.coef_ * weight
B_taus[0][i, ancestor_indexes] = coef[:causal_order_no]
for j in range(len(B_taus[1:])):
B_taus[j + 1][i, :] = coef[causal_order_no +
n_features * j:causal_order_no +
n_features * j + n_features]
return B_taus
def LassoLarsIC_df(X, y, criterion):
"""
Passes the inputs into sklearn's LassoLarsIC model selection function.
Returns the rss, intercept and all coefficients as a DataFrame
as well as a list containing the features with non-zero coefficients.
"""
model = LassoLarsIC(criterion=criterion)
model.fit(X, y.iloc[:, 0])
model_rss = model.score(X, y.iloc[:, 0])
results = ([model_rss] + [model.intercept_] + list(model.coef_))
results_cols = ['rss', 'intercept'] + list(X.columns)
results_dict = {'results': results}
results_df = pd.DataFrame.from_dict(results_dict,
orient="index",
columns=results_cols)
remaining_features = list(
results_df.iloc[0][results_df.iloc[0] != 0].index[2:])
return results_df, remaining_features
def estimate_ica(data: pd.DataFrame,
n_comp=1,
min_cluster=1,
max_cluster=30,
n_jobs=1,
random_state=1,
prefix=None):
u"""
perform ICA and using KMeans to cluster, then using BIC to estimate the best components
:param data:
:param n_comp: the number of components
:param min_cluster: see estimate_best_k
:param max_cluster: see estimate_best_k
:param prefix: see estimate_best_k
:param n_jobs: see estimate_best_k
:param random_state:
:param doARD: whether to do ARDRegression
:return:
"""
logger.info("ICA of {0}".format(n_comp))
ica = FastICA(n_components=n_comp, random_state=random_state)
S_ = ica.fit_transform(data) # Reconstruct signals
km, best_k = estimate_best_k(pd.DataFrame(S_),
min_cluster=min_cluster,
max_cluster=max_cluster,
random_state=random_state,
n_jobs=n_jobs,
prefix=prefix)
model_bic = LassoLarsIC(criterion='bic')
model_bic.fit(data, km[best_k])
return {
"n_comp": n_comp,
"ica": S_,
"km": km,
"best_k": best_k,
"bic": model_bic.alpha_
}
def _predict_adaptive_lasso(self, X, predictors, target, gamma=1.0):
"""
predict with Adaptive Lasso.
Input:
X training instances
(n_instances, n_var) numpy array
predictors indices of predictor variables
(n_predictors, ) list()
target index of target variable
int
Output:
coef Coefficients of predictor variable
(n_predictors,) numpy array
"""
lr = LinearRegression()
lr.fit(X[:, predictors], X[:, target])
weight = np.power(np.abs(lr.coef_), gamma)
reg = LassoLarsIC(criterion='bic')
reg.fit(X[:, predictors] * weight, X[:, target])
return reg.coef_ * weight
def programmer_1(inputfile, data_range):
# inputfile = "data/data1.csv"
data = pd.read_csv(inputfile)
"""
原始方法，替代方法可以使用describe()方法，然后进行筛选
r = [data.min(), data.max(), data.mean(), data.std()]
r = pd.DataFrame(r, index = ["Min", "Max", "Mean", "STD"]).T
"""
r = pd.DataFrame(data.describe()).T
np.round(r, 2)
# 计算相关系数矩阵
np.round(data.corr(method="pearson"), 2)
"""
原代码使用的是AdaptiveLasso，现更新为Lasso
参数也由gamma变为tol（有待验证）
"""
model = LassoLarsIC(criterion='aic')
model.fit(data.iloc[:, 0:data_range], data["y"])
# 各个特征的系数
# model.coef_
print(model.coef_)
def parameter_select(X, y):
print X.shape, y.shape
##############################################################################
# LassoLarsIC: least angle regression with BIC/AIC criterion
# model_bic = LassoLarsIC(criterion='bic')
# model_bic.fit(X, y)
# alpha_bic_ = model_bic.alpha_
model_aic = LassoLarsIC(criterion='aic', max_iter=100000000)
model_aic.fit(X, y)
alpha_aic_ = model_aic.alpha_
print alpha_aic_
def plot_ic_criterion(model, name, color):
alpha_ = model.alpha_
alphas_ = model.alphas_
criterion_ = model.criterion_
plt.plot(-np.log10(alphas_),
criterion_,
'--',
color=color,
linewidth=3,
label='%s criterion' % name)
plt.axvline(-np.log10(alpha_),
color=color,
linewidth=3,
label='alpha: %s estimate' % name)
plt.xlabel('-log(alpha)')
plt.ylabel('criterion')
plt.figure()
plot_ic_criterion(model_aic, 'AIC', 'b')
# plot_ic_criterion(model_bic, 'BIC', 'r')
plt.legend()
plt.title('Information-criterion for model selection')
plt.show()
fields = iot.read_fields()
for i in xrange(len(fields)):
print str(fields[i]) + '\t' + str(model_aic.coef_[i])
def regularization_lassoAIC(X, y):
"""Performs Lasso model fit with Lars using AIC for model selection.
Dependent variable = effect size within subjects; independent variable = factors.
P independent variables and n observations.
model: XB = y; y (nx1) column vector of dependent variables, X (nxP) matrix
of independent variables, B (Px1) column vector of coefficients.
Parameters
----------
X: pandas.DataFrame
Preprocessed dataframe containing all observations in rows and factors in columns (the independent variables).
Factors with too many missing values and with too many identical
observations have been removed. Besides, values have been standardized.
Categorical variables are coded in dummies.
This dataframe is obtained thanks to the ``preprocess_factors`` function.
y: pandas.Series
Effect size within subjects computed for each observation (the dependent variable) obtained after the outlier rejection.
Returns
-------
coeff_aic: pandas.DataFrame
Results of the Lasso.
Column with coefficients obtained after regularization and the names of the associated factors.
"""
model = LassoLarsIC(criterion='aic')
model.fit(X, y)
coeff_aic = pd.DataFrame({
'Factors': X.columns,
'Coefficients': model.coef_
})
return coeff_aic
def predict_adaptive_lasso(X, predictors, target, gamma=1.0):
"""Predict with Adaptive Lasso.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data, where n_samples is the number of samples
and n_features is the number of features.
predictors : array-like, shape (n_predictors)
Indices of predictor variable.
target : int
Index of target variable.
Returns
-------
coef : array-like, shape (n_features)
Coefficients of predictor variable.
"""
lr = LinearRegression()
lr.fit(X[:, predictors], X[:, target])
weight = np.power(np.abs(lr.coef_), gamma)
reg = LassoLarsIC(criterion='bic')
reg.fit(X[:, predictors] * weight, X[:, target])
return reg.coef_ * weight
def run_model(dataset, window_size, first_eday, last_eday, param, file_lambd,
file_betas):
#param: lambd for HP, level for wavelet
raw_data = np.genfromtxt(f'DATA/{dataset}.txt')
qs_real = np.reshape(raw_data[:, 2], (-1, 24))
num_days = qs_real.shape[0]
qs_predictions = np.zeros(qs_real.shape)
dummies_all = get_dummies(raw_data[:, 0])
lprices = np.log(raw_data[:, 2])
lloads = np.log(raw_data[:, 4])
for day in range(window_size + 1, num_days + 1):
first_day_index = day - (window_size + 1)
dummies = get_dummies_inwindow(dummies_all, window_size,
first_day_index)
#qs, qsmin, qsmax, zt, Ts_hat=decomposition_wavelet(lprices, lloads, first_day_index, window_size,param)
#qs, qsmin, qsmax, zt, Ts_hat=decomposition_HP(lprices, lloads, first_day_index, window_size,param)
qs, qsmin, qsmax, zt, Ts_hat = remove_mean(window_size,
first_day_index, lprices,
lloads)
for hour in range(24):
X, Y, Xr = get_calibartion_dataset(qs, qsmin, qsmax, zt,
window_size, hour, dummies)
model_aic = LassoLarsIC(criterion='aic', fit_intercept=False)
fitted_model = model_aic.fit(X, Y)
params = fitted_model.coef_
est_lambd = fitted_model.alpha_
file_lambd.write(str(est_lambd) + "\n")
np.savetxt(file_betas, params.reshape(1, params.shape[0]))
#print(params)
c_prediction = make_prediction(
Xr, params, Ts_hat
) #if wavelet or HP filters are used, change Ts_hat -> Ts_hat[hour]
qs_predictions[first_day_index + window_size, hour] = c_prediction
print(f'(day,hour):\t({day},{hour}):\t{c_prediction}')
date = raw_data[0, 0].astype(int)
first_day = get_datetime(date)
WMAE, ave_num = get_WMAE(qs_real, qs_predictions, first_day, first_eday,
last_eday)
return qs_real, qs_predictions, WMAE
def recalibrate(self, Xtrain, Ytrain):
"""Function to recalibrate the LEAR model.
It uses a training (Xtrain, Ytrain) pair for recalibration
Parameters
----------
Xtrain : numpy.array
Input in training dataset. It should be of size *[n,m]* where *n* is the number of days
in the training dataset and *m* the number of input features
Ytrain : numpy.array
Output in training dataset. It should be of size *[n,24]* where *n* is the number of days
in the training dataset and 24 are the 24 prices of each day
Returns
-------
numpy.array
The prediction of day-ahead prices after recalibrating the model
"""
# # Applying Invariant, aka asinh-median transformation to the prices
[Ytrain], self.scalerY = scaling([Ytrain], 'Invariant')
# # Rescaling all inputs except dummies (7 last features)
[Xtrain_no_dummies], self.scalerX = scaling([Xtrain[:, :-7]], 'Invariant')
Xtrain[:, :-7] = Xtrain_no_dummies
self.models = {}
for h in range(24):
# Estimating lambda hyperparameter using LARS
param_model = LassoLarsIC(criterion='aic', max_iter=2500)
param = param_model.fit(Xtrain, Ytrain[:, h]).alpha_
# Re-calibrating LEAR using standard LASSO estimation technique
model = Lasso(max_iter=2500, alpha=param)
model.fit(Xtrain, Ytrain[:, h])
self.models[h] = model
def fit_models_LassoCV(self, X, Y, bands=None):
""" Try to fit models to training period time series """
if bands is None:
bands = self.fit_indices
models = []
for b in bands:
# lasso = LassoCV(n_alphas=100)
# lasso = LassoLarsCV(masx_n_alphas=100)
lasso = LassoLarsIC(criterion='bic')
lasso = lasso.fit(X, Y[b, :])
lasso.nobs = Y[b, :].size
lasso.coef = np.copy(lasso.coef_)
lasso.coef[0] += lasso.intercept_
lasso.fittedvalues = lasso.predict(X)
lasso.rss = np.sum((Y[b, :] - lasso.fittedvalues) ** 2)
lasso.rmse = math.sqrt(lasso.rss / lasso.nobs)
models.append(lasso)
return np.array(models)
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
rng = np.random.RandomState(42)
X = np.c_[X, rng.randn(X.shape[0], 14)] # add some bad features
# normalize data as done by Lars to allow for comparison
X /= np.sqrt(np.sum(X**2, axis=0))
# #############################################################################
# LassoLarsIC: least angle regression with BIC/AIC criterion
model_bic = LassoLarsIC(criterion='bic')
t1 = time.time()
model_bic.fit(X, y)
t_bic = time.time() - t1
alpha_bic_ = model_bic.alpha_
model_aic = LassoLarsIC(criterion='aic')
model_aic.fit(X, y)
alpha_aic_ = model_aic.alpha_
def plot_ic_criterion(model, name, color):
alpha_ = model.alpha_ + EPSILON
alphas_ = model.alphas_ + EPSILON
criterion_ = model.criterion_
plt.plot(-np.log10(alphas_),
criterion_,
'--',
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
rng = np.random.RandomState(42)
X = np.c_[X, rng.randn(X.shape[0], 14)] # add some bad features
# normalize data as done by Lars to allow for comparison
X /= np.sqrt(np.sum(X ** 2, axis=0))
##############################################################################
# LassoLarsIC: least angle regression with BIC/AIC criterion
model_bic = LassoLarsIC(criterion="bic")
t1 = time.time()
model_bic.fit(X, y)
t_bic = time.time() - t1
alpha_bic_ = model_bic.alpha_
model_aic = LassoLarsIC(criterion="aic")
model_aic.fit(X, y)
alpha_aic_ = model_aic.alpha_
def plot_ic_criterion(model, name, color):
alpha_ = model.alpha_
alphas_ = model.alphas_
criterion_ = model.criterion_
plt.plot(-np.log10(alphas_), criterion_, "--", color=color, linewidth=3, label="%s criterion" % name)
plt.axvline(-np.log10(alpha_), color=color, linewidth=3, label="alpha: %s estimate" % name)
plt.xlabel("-log(alpha)")
coefs.append(model.coef_)
plt.figure(figsize = (12,6))
ax = plt.gca()
ax.plot(alphas*2, coefs)
ax.set_xscale("log")
plt.axis("tight")
plt.xlabel("alpha")
plt.ylabel("weights");
plt.title("Different lambda regularization values for Lasso (alpha in Python)")
plt.show()
# Initiate BIC and AIC alphas for Lasso regression
model_bic = LassoLarsIC(criterion = "bic", normalize = True, max_iter = 10000)
t1 = time.time()
model_bic.fit(X_train, y_train)
t_bic = time.time() - t1
alpha_bic_ = model_bic.alpha_
model_aic = LassoLarsIC(criterion = "aic", normalize = True, max_iter = 10000)
model_aic.fit(X_train, y_train)
alpha_aic_ = model_aic.alpha_
# Plot Aikake's and Bayesian information criterion for model selection
def plot_ic_criterion(model, name, color):
alpha_ = model.alpha_
alphas_ = model.alphas_
criterion_ = model.criterion_
plt.plot(-np.log10(alphas_), criterion_, "--", color = color,
linewidth = 3, label = "%s criterion" % name)
plt.axvline(-np.log10(alpha_), color=color, linewidth = 3,
def solve(self, fraction_evaluated):
count = 0.0
for i in range(self.maskMatrix.shape[0]):
if self.maskMatrix[i, 0] == 1 and sum(self.maskMatrix[i, 1:]) == 0:
count += 1
log.info("[1,0,0,0] ratio = {0}".format(count /
self.maskMatrix.shape[0]))
count = 0.0
for i in range(self.maskMatrix.shape[0]):
if sum(self.maskMatrix[i, :]) == 2 or sum(
self.maskMatrix[i, :]) == 18:
count += 1
log.info("2 or 18 sum ratio = {0}".format(count /
self.maskMatrix.shape[0]))
count = 0.0
for i in range(self.maskMatrix.shape[0]):
if sum(self.maskMatrix[i, :]) == 3 or sum(
self.maskMatrix[i, :]) == 17:
count += 1
log.info("3 or 17 sum ratio = {0}".format(count /
self.maskMatrix.shape[0]))
count = 0.0
for i in range(self.maskMatrix.shape[0]):
if sum(self.maskMatrix[i, :]) == 0:
count += 1
log.info("0 sum ratio = {0}".format(count / self.maskMatrix.shape[0]))
count = 0.0
for i in range(self.maskMatrix.shape[0]):
if sum(self.maskMatrix[i, :]) == 10:
count += 1
log.info("10 sum ratio = {0}".format(count / self.maskMatrix.shape[0]))
# self.maskMatrix = self.maskMatrix[:self.nsamplesAdded,:]
# self.ey = self.ey[:self.nsamplesAdded]
# self.kernelWeights = self.kernelWeights[:self.nsamplesAdded]
log.debug("self.maskMatrix.shape = {0}".format(self.maskMatrix.shape))
# adjust the y value according to the constraints for the offset and sum
log.debug("self.link(self.fnull) = {0}".format(self.link.f(
self.fnull)))
log.debug("self.link(self.fx) = {0}".format(self.link.f(self.fx)))
# for i in range(self.maskMatrix.shape[0]):
# log.debug("{0} {1} {2}".format(self.maskMatrix[i,:], self.ey[i], self.kernelWeights[i]))
eyAdj = self.linkfv(self.ey) - self.link.f(self.fnull)
s = np.sum(self.maskMatrix, 1)
# do feature selection if we have not well enumerated the space
nonzero_inds = np.arange(self.M)
if fraction_evaluated < 0.2:
w_aug = np.hstack(
(self.kernelWeights * (self.M - s), self.kernelWeights * s))
log.info("np.sum(w_aug) = {0}".format(np.sum(w_aug)))
log.info("np.sum(self.kernelWeights) = {0}".format(
np.sum(self.kernelWeights)))
w_sqrt_aug = np.sqrt(w_aug)
eyAdj_aug = np.hstack(
(eyAdj,
eyAdj - (self.link.f(self.fx) - self.link.f(self.fnull))))
eyAdj_aug *= w_sqrt_aug
mask_aug = np.transpose(
w_sqrt_aug *
np.transpose(np.vstack(
(self.maskMatrix, self.maskMatrix - 1))))
var_norms = np.array([
np.linalg.norm(mask_aug[:, i])
for i in range(mask_aug.shape[1])
])
#mask_aug /= var_norms
# print(self.kernelWeights)
# print(w_aug)
model = LassoLarsIC(criterion='bic',
normalize=True) #fit_intercept
#model = Lasso(alpha=self.l1reg, fit_intercept=True)
model.fit(mask_aug, eyAdj_aug)
nonzero_inds = np.nonzero(model.coef_)[0]
# for i in range(mask_aug.shape[0]):
# log.info("{0} {1} {2}".format(mask_aug[i,:], self.ey[i], self.kernelWeights[i]))
log.info("model.get_params() = {0}".format(model.get_params()))
#log.info("model.alpha_ = {0}".format(model.alpha_))
log.info("model.coef_ = {0}".format(model.coef_))
log.info("nonzero_inds = {0}".format(nonzero_inds))
w1 = np.dot(
np.linalg.inv(np.dot(np.transpose(mask_aug), mask_aug)),
np.dot(np.transpose(mask_aug), eyAdj_aug))
log.info("w1 = {0}".format(w1))
w1 = np.dot(
np.linalg.inv(np.dot(np.transpose(mask_aug), mask_aug)),
np.dot(np.transpose(mask_aug), eyAdj_aug))
log.info("w1 = {0}".format(w1))
#np.transpose(self.maskMatrix) * self.kernelWeights
#w = np.dot(np.linalg.inv(np.dot(np.transpose(mask_aug),mask_aug)),np.dot(np.transpose(mask_aug), eyAdj_aug))
# eyAdj1 = eyAdj - self.maskMatrix[:,-1]*(self.link(self.fx) - self.link(self.fnull))
# etmp = self.maskMatrix[:,:-1] - self.maskMatrix[:,-1:]
# var_norms = np.array([np.linalg.norm(etmp[:,i]) for i in range(etmp.shape[1])])
# etmp /= var_norms
# print(var_norms)
# model_bic = LassoLarsIC(criterion='bic')
# model_bic.fit(etmp, eyAdj1)
# nonzero_inds = np.nonzero(model_bic.coef_)[0]
# print(nonzero_inds.shape)
# # solve a weighted least squares equation to estimate phi
# print(self.maskMatrix[:,nonzero_inds[-1]].shape)
# print(nonzero_inds)
#nonzero_inds = np.arange(self.M)
eyAdj2 = eyAdj - self.maskMatrix[:, nonzero_inds[-1]] * (
self.link.f(self.fx) - self.link.f(self.fnull))
etmp = np.transpose(
np.transpose(self.maskMatrix[:, nonzero_inds[:-1]]) -
self.maskMatrix[:, nonzero_inds[-1]])
#print(self.maskMatrix)
log.debug("etmp[1:4,:] {0}".format(etmp[0:4, :]))
# etmp = self.maskMatrix
# eyAdj2 = eyAdj
# solve a weighted least squares equation to estimate phi
tmp = np.transpose(
np.transpose(etmp) * np.transpose(self.kernelWeights))
#tmp = etmp
# log.debug("tmp.shape", tmp.shape)
# log.debug("tmp.shape", tmp.shape)
tmp2 = np.linalg.inv(np.dot(np.transpose(tmp), etmp))
w = np.dot(tmp2, np.dot(np.transpose(tmp), eyAdj2))
#log.info("w = {0}".format(w))
log.debug("np.sum(w) = {0}".format(np.sum(w)))
log.debug("self.link(self.fx) - self.link(self.fnull) = {0}".format(
self.link.f(self.fx) - self.link.f(self.fnull)))
phi = np.zeros(self.M)
phi[nonzero_inds[:-1]] = w
phi[nonzero_inds[-1]] = (self.link.f(self.fx) -
self.link.f(self.fnull)) - sum(w)
log.info("phi = {0}".format(phi))
# clean up any rounding errors
for i in range(self.M):
if np.abs(phi[i]) < 1e-10:
phi[i] = 0
# yHat = np.dot(self.maskMatrix, w)
# phi_var = np.var(yHat - eyAdj) * np.diag(tmp2)
# phi_var = np.hstack((phi_var, max(phi_var))) # since the last weight is inferred we use a pessimistic guess of its variance
# a finite sample adjustment based on how much of the weight is left in the sample space
# fractionWeightLeft = 1 - sum(self.kernelWeights)/sum(np.array([(self.M-1)/(s*(self.M-s)) for s in range(1, self.M)]))
return phi, np.ones(len(phi)) #phi_var*fractionWeightLeft
rsquared_train=model.score(pred_train,tar_train)
rsquared_test=model.score(pred_test,tar_test)
print ('training data R-square')
print(rsquared_train)
print ('test data R-square')
print(rsquared_test)
##
from sklearn.linear_model import LassoCV, LassoLarsCV, LassoLarsIC
##next we try and fit using AIC criterion
model_aic = LassoLarsIC(criterion='aic')
model_aic.fit(pred_train,tar_train)
alpha_aic_ = model_aic.alpha_
def plot_ic_criterion(model, name, color):
alpha_ = model.alpha_
alphas_ = model.alphas_
criterion_ = model.criterion_
plt.plot(-np.log10(alphas_), criterion_, '--', color=color,
linewidth=3, label='%s criterion' % name)
plt.axvline(-np.log10(alpha_), color=color, linewidth=3,
label='alpha: %s estimate' % name)
plt.xlabel('-log(alpha)')
plt.ylabel('criterion')
plt.figure()
plot_ic_criterion(model_aic, 'AIC', 'b')
def trainData(fileName):
df = pd.read_csv(fileName, index_col='date')
df = df.sort_index()
df = df[[
'open', 'high', 'close', 'low', 'volume', 'price_change', 'p_change',
'ma5', 'ma10', 'ma20', 'v_ma5', 'v_ma10', 'v_ma20', 'turnover'
]]
df = df[['open', 'high', 'low', 'close', 'volume']]
df['HL_PCT'] = (df['high'] - df['low']) / df['close'] * 100.0
df['PCT_change'] = (df['close'] - df['open']) / df['open'] * 100.0
df = df[['close', 'HL_PCT', 'PCT_change', 'volume']]
# print(df.head())
forecast_col = 'close'
df.fillna(value=-99999, inplace=True)
# forecast_out = int(math.ceil(0.01 * len(df)))
forecast_out = 1
# ??forecast_out???
df['label'] = df[forecast_col].shift(-forecast_out)
print(df.shape)
print(df)
X = np.array(df.drop(['label'], 1))
X = preprocessing.scale(X)
X_lately = X[-forecast_out:]
X = X[:-forecast_out]
df.dropna(inplace=True)
print(X)
print(X_lately)
y = np.array(df['label'])
# print(y)
print(X.shape)
print(y.shape)
X_train, X_test, y_train, y_test = cross_validation.train_test_split(
X, y, test_size=0.2)
clf = LassoLarsIC(max_iter=100)
clf.fit(X_train, y_train)
accuracy = clf.score(X_test, y_test)
joblib.dump(clf, "%s.m" % fileName)
print(accuracy, "---------score------")
forecast_set = clf.predict(X_lately)
print(forecast_out)
style.use('ggplot')
df['Forecast'] = np.nan
last_date = df.iloc[-1].name
date_time = datetime.datetime.strptime(last_date, '%Y-%m-%d')
last_unix = date_time.timestamp()
one_day = 86400
next_unix = last_unix + one_day
print(forecast_set)
for i in forecast_set:
next_date = datetime.datetime.fromtimestamp(next_unix)
next_unix += 86400
df.loc[next_date] = [np.nan for _ in range(len(df.columns) - 1)] + [i]
print(df.tail(forecast_out))
df['close'].plot()
df['Forecast'].plot()
plt.show()
'--',
color=color,
linewidth=3,
label='%s criterion' % name)
plt.axvline(alpha_,
color=color,
linewidth=3,
label='alpha: %s estimate' % name)
plt.xlabel('alpha')
plt.ylabel('criterion')
#Choose the optimal alpha for LASSO regression using the AIC & BIC criterion
model_bic = LassoLarsIC(criterion='bic')
t1 = time.time()
model_bic.fit(X, Y)
t_bic = time.time() - t1
alpha_bic_ = model_bic.alpha_
model_aic = LassoLarsIC(criterion='aic')
model_aic.fit(X, Y)
alpha_aic_ = model_aic.alpha_
plt.figure()
plot_ic_criterion(model_aic, 'AIC', 'b')
plot_ic_criterion(model_bic, 'BIC', 'r')
plt.legend()
plt.title('Information-criterion for model selection (training time %.3fs)' %
t_bic)
#Use the Randomized LASSO module to choose the best alpha representing the data
def solve(self, fraction_evaluated, dim):
eyAdj = self.linkfv(self.ey[:, dim]) - self.link.f(self.fnull[dim])
s = np.sum(self.maskMatrix, 1)
# do feature selection if we have not well enumerated the space
nonzero_inds = np.arange(self.M)
log.debug("fraction_evaluated = {0}".format(fraction_evaluated))
if (self.l1_reg not in [
"auto", False, 0
]) or (fraction_evaluated < 0.2 and self.l1_reg == "auto"):
w_aug = np.hstack(
(self.kernelWeights * (self.M - s), self.kernelWeights * s))
log.info("np.sum(w_aug) = {0}".format(np.sum(w_aug)))
log.info("np.sum(self.kernelWeights) = {0}".format(
np.sum(self.kernelWeights)))
w_sqrt_aug = np.sqrt(w_aug)
eyAdj_aug = np.hstack(
(eyAdj, eyAdj -
(self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim]))))
eyAdj_aug *= w_sqrt_aug
mask_aug = np.transpose(
w_sqrt_aug *
np.transpose(np.vstack(
(self.maskMatrix, self.maskMatrix - 1))))
var_norms = np.array([
np.linalg.norm(mask_aug[:, i])
for i in range(mask_aug.shape[1])
])
if self.l1_reg == "auto":
model = LassoLarsIC(criterion="aic")
elif self.l1_reg == "bic" or self.l1_reg == "aic":
model = LassoLarsIC(criterion=self.l1_reg)
else:
model = Lasso(alpha=self.l1_reg)
model.fit(mask_aug, eyAdj_aug)
nonzero_inds = np.nonzero(model.coef_)[0]
if len(nonzero_inds) == 0:
return np.zeros(self.M), np.ones(self.M)
# eliminate one variable with the constraint that all features sum to the output
eyAdj2 = eyAdj - self.maskMatrix[:, nonzero_inds[-1]] * (
self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim]))
etmp = np.transpose(
np.transpose(self.maskMatrix[:, nonzero_inds[:-1]]) -
self.maskMatrix[:, nonzero_inds[-1]])
log.debug("etmp[:4,:] {0}".format(etmp[:4, :]))
# solve a weighted least squares equation to estimate phi
tmp = np.transpose(
np.transpose(etmp) * np.transpose(self.kernelWeights))
tmp2 = np.linalg.inv(np.dot(np.transpose(tmp), etmp))
w = np.dot(tmp2, np.dot(np.transpose(tmp), eyAdj2))
log.debug("np.sum(w) = {0}".format(np.sum(w)))
log.debug("self.link(self.fx) - self.link(self.fnull) = {0}".format(
self.link.f(self.fx[dim]) - self.link.f(self.fnull[dim])))
log.debug("self.fx = {0}".format(self.fx[dim]))
log.debug("self.link(self.fx) = {0}".format(self.link.f(self.fx[dim])))
log.debug("self.fnull = {0}".format(self.fnull[dim]))
log.debug("self.link(self.fnull) = {0}".format(
self.link.f(self.fnull[dim])))
phi = np.zeros(self.M)
phi[nonzero_inds[:-1]] = w
phi[nonzero_inds[-1]] = (self.link.f(self.fx[dim]) -
self.link.f(self.fnull[dim])) - sum(w)
log.info("phi = {0}".format(phi))
# clean up any rounding errors
for i in range(self.M):
if np.abs(phi[i]) < 1e-10:
phi[i] = 0
return phi, np.ones(len(phi))
def Plot_Lasso_LARS_Path(self):
X = self.X
y = self.Y
# normalize data as done by Lars to allow for comparison
X /= np.sqrt(np.sum(X ** 2, axis=0))
#######################################################################
# LassoLarsIC: least angle regression with BIC/AIC criterion
model_bic = LassoLarsIC(criterion='bic')
t1 = time.time()
model_bic.fit(X, y)
t_bic = time.time() - t1
model_aic = LassoLarsIC(criterion='aic')
model_aic.fit(X, y)
plt.figure()
self.plot_ic_criterion(model_aic, 'AIC', 'b')
self.plot_ic_criterion(model_bic, 'BIC', 'r')
print model_bic.alpha_
plt.legend()
plt.title('Information-criterion for model selection (training time %.3fs)'
% t_bic)
#######################################################################
# LassoCV: coordinate descent
# Compute paths
print(
"Computing regularization path using the coordinate descent lasso...")
t1 = time.time()
model = LassoCV(cv=20).fit(X, y)
t_lasso_cv = time.time() - t1
# Display results
m_log_alphas = -np.log10(model.alphas_)
plt.figure()
ymin, ymax = 2300, 3800
plt.plot(m_log_alphas, model.mse_path_, ':')
plt.plot(m_log_alphas, model.mse_path_.mean(axis=-1), 'k',
label='Average across the folds', linewidth=2)
plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k',
label='alpha: CV estimate')
plt.legend()
plt.xlabel('-log(alpha)')
plt.ylabel('Mean square error')
plt.title('Mean square error on each fold: coordinate descent '
'(train time: %.2fs)' % t_lasso_cv)
plt.axis('tight')
plt.ylim(ymin, ymax)
#######################################################################
# LassoLarsCV: least angle regression
# Compute paths
print("Computing regularization path using the Lars lasso...")
t1 = time.time()
model = LassoLarsCV(cv=20).fit(X, y)
t_lasso_lars_cv = time.time() - t1
# Display results
m_log_alphas = -np.log10(model.cv_alphas_)
plt.figure()
plt.plot(m_log_alphas, model.cv_mse_path_, ':')
plt.plot(m_log_alphas, model.cv_mse_path_.mean(axis=-1), 'k',
label='Average across the folds', linewidth=2)
plt.axvline(-np.log10(model.alpha_), linestyle='--', color='k',
label='alpha CV')
plt.legend()
plt.xlabel('-log(alpha)')
plt.ylabel('Mean square error')
plt.title('Mean square error on each fold: Lars (train time: %.2fs)'
% t_lasso_lars_cv)
plt.axis('tight')
plt.ylim(ymin, ymax)
plt.show()
def detect_stops_on_link_traj(traj, debug=False, **param):
"""
Detects if a trajectory (characterized by a list of tspots on a link) corresponds to a stopping vehicle
Input:
traj: list of tspots on a link
debug (optional): if True, returns extra variables to help with debugging. Default=False
Output:
stopping (bool): if True, indicates that there is a stop on the trajectory
If debug is set to True (default=False)
obs: list of Point_pts representing the
offset and time (in seconds after beginning of traj)
of the measurements
est: list of Point_pts representing the
offset and time (in seconds after beginning of traj)
of the estimated location of the measurements
"""
if len(traj) <= 3:
return False
loc = np.array([x.spot.offset - traj[0].spot.offset for x in traj[1 :]])
time = np.array([(x.time - traj[0].time).total_seconds() for x in traj])
n = len(time) - 1
A = np.zeros((n, n), dtype=np.float64)
for i in range(n):
A[i :, i] = time[i + 1] - time[i]
model_bic = LassoLarsIC(criterion='bic', fit_intercept=False)
# There is a bug here for the following data:
#[[ 0.669923 0. 0. 0. 0. 0. 0.
# 0. ]
# [ 0.669923 2. 0. 0. 0. 0. 0.
# 0. ]
# [ 0.669923 2. 34. 0. 0. 0. 0.
# 0. ]
# [ 0.669923 2. 34. 2. 0. 0. 0.
# 0. ]
# [ 0.669923 2. 34. 2. 2. 0. 0.
# 0. ]
# [ 0.669923 2. 34. 2. 2. 4. 0.
# 0. ]
# [ 0.669923 2. 34. 2. 2. 4. 2.
# 0. ]
# [ 0.669923 2. 34. 2. 2. 4. 2.
# 0.94968 ]]
#[ 6.24743444 6.24743444 6.24743444 10.41858373 14.46159935
# 26.17648665 39.90241795 52.77 ]
#
# if A.shape == (8,8):
# print A
# print loc
try:
model_bic.fit(A, loc)
except TypeError:
print "Failure in detect_stops_on_link_traj"
return False
if debug:
print 'Lasso BIC'
print np.dot(A, model_bic.coef_)
print model_bic.coef_
stop_time = 0
stopping = False
for i, speed in enumerate(model_bic.coef_):
if speed < param['speed_threshold']:
stop_time += time[i + 1] - time[i]
if stop_time >= param['min_stop_duration']:
stopping = True
break
if not debug:
return stopping
est_loc = [0.0] + list(np.dot(A, model_bic.coef_))
obs = [struct.Point_pts(s.spot.offset, t, 0) for (s, t) in zip(traj, time)]
est = [struct.Point_pts(e + traj[0].spot.offset, t, 0) for (e, t) in zip(est_loc, time)]
plt.plot(time[1 :], loc, '-+b', linewidth=5, label='Observations')
plt.plot(time, est_loc, '-or', label='Estimate')
print stopping
plt.legend()
plt.show()
return (stopping, obs, est)
import time
import matplotlib.pyplot as plt
from sklearn.linear_model import LassoCV, LassoLarsCV, LassoLarsIC
##############################################################################
# LassoLarsIC: least angle regression with BIC/AIC criterion
model_bic = LassoLarsIC(criterion='bic')
t1 = time.time()
model_bic.fit(X_train, y_train)
t_bic = time.time() - t1
alpha_bic_ = model_bic.alpha_
model_aic = LassoLarsIC(criterion='aic')
model_aic.fit(X_train, y_train)
alpha_aic_ = model_aic.alpha_
def plot_ic_criterion(model, name, color):
alpha_ = model.alpha_
alphas_ = model.alphas_
criterion_ = model.criterion_
plt.plot(-np.log10(alphas_), criterion_, '--', color=color,
linewidth=3, label='%s criterion' % name)
plt.axvline(-np.log10(alpha_), color=color, linewidth=3,
# Train the model using the training sets
regrLasso.fit(X_train, Y_train)
# Make predictions using the testing set
Y_predLasso = regrLasso.predict(X_test)
# The coefficients
print('Coefficients: \n', regrLasso.coef_)
# The mean squared error
print("Mean squared error: %.2f" % mean_squared_error(Y_test, Y_predLasso))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.2f' % r2_score(Y_test, Y_predLasso))
# #############################################################################
# LassoLarsIC: least angle regression with BIC criterion
model_bic = LassoLarsIC(criterion='bic')
model_bic.fit(X_train, Y_train)
alpha_bic_ = model_bic.alpha_
def plot_ic_criterion(model, name, color):
alpha_ = model.alpha_
alphas_ = model.alphas_
criterion_ = model.criterion_
plt.plot(-np.log10(alphas_),
criterion_,
'--',
color=color,
linewidth=3,
label='%s criterion' % name)
plt.axvline(-np.log10(alpha_),
color=color,
