def dump(self, dictionary, path, fitter=None):
bases = dictionary.shape[1]
assert (bases < dictionary.shape[0])
backup_fitter = lm.OrthogonalMatchingPursuit(4)
if fitter is None:
fitter = backup_fitter
coefs = []
bad_samples = 0
for f in self.files:
x = self.load_sample(f).T
fitter.fit(dictionary, x)
b = None
if absmax(fitter.coef_) != 0:
b = np.copy(fitter.coef_)
else:
backup_fitter.fit(dictionary, x)
if absmax(backup_fitter.coef_) != 0:
b = np.copy(backup_fitter.coef_)
if b is not None:
coefs.append(b)
else:
bad_samples += 1
X = make_mxe_loadable(dictionary.T.astype(np.float32), 16)
Y = np.array(coefs).astype(np.float32)
np.save(path, X)
np.save(path + " coefficients", Y)
def get_regression_models():
models = [('LR', linear_model.LinearRegression()),
('R', linear_model.Ridge()),
('Lo', linear_model.Lasso(alpha=.015)),
('La', linear_model.Lars(positive=True)),
('OMP', linear_model.OrthogonalMatchingPursuit()),
('BR', linear_model.BayesianRidge()),
('GB',
ensemble.GradientBoostingRegressor(alpha=0.9,
criterion='friedman_mse',
init=None,
learning_rate=0.1,
loss='ls',
max_depth=5,
max_features=None,
min_samples_leaf=1,
min_samples_split=2,
min_weight_fraction_leaf=.0,
n_estimators=400,
presort='auto',
random_state=None,
subsample=1.0,
verbose=0,
warm_start=False)),
('RF', ensemble.RandomForestRegressor()),
('AB', ensemble.AdaBoostRegressor())]
return models
def test_sk_OrthogonalMatchingPursuit():
print("Testing sklearn, OrthogonalMatchingPursuit...")
mod = linear_model.OrthogonalMatchingPursuit()
X, y = iris_data
mod.fit(X, y)
docs = {'name': "OrthogonalMatchingPursuit test"}
fv = X[0, :]
upload(mod, fv, docs)
def orthogonalMatchingPursuit(self):
omp = linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=10)
omp.fit(self.training_order_start_end_districts_and_time,
self.training_number_of_orders)
predicted_number_of_orders = omp.predict(
self.testing_order_start_end_districts_and_time)
current_ride_prediction_error = numpy.mean(
(predicted_number_of_orders - self.testing_number_of_orders)**2)
print(current_ride_prediction_error)
print(omp.coef_)
def _train(self):
x = self._train_set.features
y = self._train_set.outputs
self._transform = preprocessing.PolynomialFeatures(3)
clf = linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=235,
fit_intercept=True)
clf.fit(self._transform.fit_transform(x, y), y)
self._model = clf.predict
def __init__(self, method, params, i=0):
self.algorithm_list = [
'PLS', 'GP', 'OLS', 'OMP', 'Lasso', 'Elastic Net', 'Ridge',
'Bayesian Ridge', 'ARD', 'LARS', 'LASSO LARS', 'SVR', 'KRR', 'GBR'
]
self.method = method
self.outliers = None
self.ransac = False
#print(params)
if self.method[i] == 'PLS':
self.model = PLSRegression(**params[i])
if self.method[i] == 'OLS':
self.model = linear.LinearRegression(**params[i])
if self.method[i] == 'OMP':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
self.model = linear.OrthogonalMatchingPursuit(**params_temp)
if self.method[i] == 'LASSO':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
self.model = linear.Lasso(**params_temp)
if self.method[i] == 'Elastic Net':
params_temp = copy.copy(params[i])
self.model = linear.ElasticNet(**params_temp)
if self.method[i] == 'Ridge':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
self.model = linear.Ridge(**params_temp)
if self.method[i] == 'BRR':
self.model = linear.BayesianRidge(**params[i])
if self.method[i] == 'ARD':
self.model = linear.ARDRegression(**params[i])
if self.method[i] == 'LARS':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
self.model = linear.Lars(**params_temp)
if self.method[i] == 'SVR':
self.model = svm.SVR(**params[i])
if self.method[i] == 'KRR':
self.model = kernel_ridge.KernelRidge(**params[i])
def test_model_orthogonal_matching_pursuit(self):
model, X = fit_regression_model(
linear_model.OrthogonalMatchingPursuit())
model_onnx = convert_sklearn(
model,
"orthogonal matching pursuit",
[("input", FloatTensorType([None, X.shape[1]]))],
target_opset=TARGET_OPSET)
self.assertIsNotNone(model_onnx)
dump_data_and_model(X,
model,
model_onnx,
verbose=False,
basename="SklearnOrthogonalMatchingPursuit-Dec4")
def sklearn_liner_model_regressions(xTrain, xTest, yTrain, yTest):
modelForConsideration: DataFrame = pd.DataFrame()
LinerModels = \
[
linear_model.ARDRegression(), linear_model.BayesianRidge(), linear_model.ElasticNet(),
linear_model.ElasticNetCV(),
linear_model.HuberRegressor(), linear_model.Lars(), linear_model.LarsCV(), linear_model.Lasso(),
linear_model.LassoCV(), linear_model.LassoLars(), linear_model.LassoLarsCV(), linear_model.LassoLarsIC(),
linear_model.LinearRegression(), linear_model.MultiTaskLasso(),
linear_model.MultiTaskElasticNet(), linear_model.MultiTaskLassoCV(), linear_model.MultiTaskElasticNetCV(),
linear_model.OrthogonalMatchingPursuit(),
linear_model.OrthogonalMatchingPursuitCV(), linear_model.PassiveAggressiveClassifier(),
linear_model.PassiveAggressiveRegressor(), linear_model.Perceptron(),
linear_model.RANSACRegressor(), linear_model.Ridge(), linear_model.RidgeClassifier(),
linear_model.RidgeClassifierCV(),
linear_model.RidgeCV(), linear_model.SGDClassifier(), linear_model.SGDRegressor(),
linear_model.TheilSenRegressor(),
linear_model.enet_path(xTrain, yTrain),
linear_model.lars_path(xTrain, yTrain), linear_model.lasso_path(xTrain, yTrain),
# linear_model.LogisticRegression()
# ,linear_model.LogisticRegressionCV(),linear_model.logistic_regression_path(xTrain, yTrain), linear_model.orthogonal_mp(xTrain, yTrain), linear_model.orthogonal_mp_gram(), linear_model.ridge_regression()
]
for model in LinerModels:
modelName: str = model.__class__.__name__
try:
# print(f"Preparing Model {modelName}")
if modelName == "LogisticRegression":
model = linear_model.LogisticRegression(random_state=0)
model.fit(xTrain, yTrain)
yTrainPredict = model.predict(xTrain)
yTestPredict = model.predict(xTest)
errorList = calculate_prediction_error(modelName, yTestPredict,
yTest, yTrainPredict,
yTrain)
if errorList["Test Average Error"][0] < 30 and errorList[
"Train Average Error"][0] < 30:
try:
modelForConsideration = modelForConsideration.append(
errorList)
except (Exception) as e:
print(e)
except (Exception, ArithmeticError) as e:
print(f"Error occurred while preparing Model {modelName}")
return modelForConsideration
def test_model_orthogonal_matching_pursuit(self):
model, X = fit_regression_model(
linear_model.OrthogonalMatchingPursuit())
model_onnx = convert_sklearn(
model, "orthogonal matching pursuit",
[("input", FloatTensorType([None, X.shape[1]]))])
self.assertIsNotNone(model_onnx)
dump_data_and_model(
X,
model,
model_onnx,
verbose=False,
basename="SklearnOrthogonalMatchingPursuit-Dec4",
allow_failure="StrictVersion("
"onnxruntime.__version__)"
"<= StrictVersion('0.2.1')",
)
def dtc08(self):
#将y转化为一维形式：self.y_train，self.y_test
self.y01_train = list()
self.y01_test = list()
for a in range(len(self.y_train)):
self.y01_train.append(self.y_train[a][0])
for b in range(len(self.y_test)):
self.y01_test.append(self.y_test[b][0])
if not self.om_edit.text().strip():
self.om_alpha = None
else:
self.om_alpha = float(self.om_edit.text())
#LR算法实现
self.clf_om = linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs = self.om_alpha)
self.clf_om.fit(self.x_train, self.y01_train)
self.y_pred = self.clf_om.predict(self.x_test)
self.x_pred = self.clf_om.predict(self.x_train)
#设置值
self.stab(self.om_table02, self.om_table03)
self.eetab(self.om_table01)
def choose_model(self, X, y):
"""
Automatic model chooser.
:param X: data
:param y: target
:type X: ndarray or scipy.sparse matrix, (n_samples, n_features)
:type y: ndarray, shape (n_samples,) or (n_samples, n_targets)
"""
#{'linear', 'polynomial',logistic','logisticcv','elasticnet','elasticnetcv','orthogonal','orthogonalcv','theil','sgd','perceptron','passive_aggressive'}
models = {
'linear': linear_model.LinearRegression(),
'logistic': linear_model.LogisticRegression(),
'elasticnet': linear_model.ElasticNet(),
'orthogonal': linear_model.OrthogonalMatchingPursuit(),
'theil': linear_model.TheilSenRegressor(),
'sgd': linear_model.SGDRegressor(),
'passive_agressive': linear_model.PassiveAggressiveRegressor()
}
scores = {}
for name, model in models.items():
scores[name] = []
sss = StratifiedShuffleSplit(10, 0.25)
for train_index, test_index in sss.split(X, y):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
for name, model in models:
mode.fit(X_train, y_train)
scores[name].append(metrics.mean_squared_error(X_test, y_test))
#Choose http://blog.minitab.com/blog/adventures-in-statistics-2/how-to-choose-the-best-regression-model
index = None
for name, model in models:
min = 10000
if scores[name][-1] < min:
min = scores[name][-1]
self._model = model
def get_stats(path):
info = pd.read_csv(path)
info = info.dropna()
f = info['price'] < 100000
info = info[f] # Get information only about flats with price < 100'000
X = info[['type', 'size', 'locality']].values
scaler_X = preprocessing.StandardScaler().fit(X)
X = scaler_X.transform(X)
y = info['price'].values
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
random_state=20)
estimators = [
linear_model.LinearRegression(),
linear_model.Ridge(alpha=0.1),
linear_model.Lasso(alpha=0.1),
linear_model.ElasticNet(alpha=0.01, l1_ratio=0.25),
linear_model.BayesianRidge(n_iter=500),
linear_model.OrthogonalMatchingPursuit(),
linear_model.SGDRegressor(max_iter=2500, epsilon=0.01),
SVR(kernel='rbf', epsilon=0.01, C=20)
]
estimator_values = np.array([])
for e in estimators:
e.fit(X_train, y_train)
this_err = metrics.median_absolute_error(y_test, e.predict(X_test))
estimator_values = np.append(estimator_values, this_err)
return estimator_values
def grid_Search(self):
'''
this function is used to evaluate various regression methods
'''
rs = 1
ests = [
linear_model.LinearRegression(),
linear_model.Ridge(),
linear_model.Lasso(),
linear_model.ElasticNet(),
linear_model.BayesianRidge(),
linear_model.OrthogonalMatchingPursuit(),
ensemble.GradientBoostingRegressor()
]
ests_labels = np.array([
'Linear', 'Ridge', 'Lasso', 'ElasticNet', 'BayesRidge', 'OMP',
'GradientBoostRegressor'
])
errvals = np.array([])
for e in ests:
e.fit(self.X_train, self.y_train)
this_err = mean_squared_error(self.y_test, e.predict(self.X_test))
#print"got error %0.2f" % this_err
errvals = np.append(errvals, math.sqrt(this_err))
pos = np.arange(errvals.shape[0])
srt = np.argsort(errvals)
plt.figure(figsize=(12, 10))
plt.bar(pos, errvals[srt], align='center')
plt.xticks(pos, ests_labels[srt])
plt.xlabel('Estimator')
plt.ylabel('Root Mean Square Error')
plt.show()
return
LINEAR_MODELS = {
"none":
None,
"linear":
linear_model.LinearRegression(fit_intercept=False),
"elastic_net":
linear_model.MultiTaskElasticNet(alpha=0.0001, fit_intercept=False),
"lasso":
linear_model.MultiTaskLasso(alpha=0.001, fit_intercept=False),
"lasso_lars":
linear_model.LassoLars(alpha=0.0001, fit_intercept=False),
"lars":
linear_model.Lars(n_nonzero_coefs=10, fit_intercept=False),
"matching_pursuit":
linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=10,
fit_intercept=False),
"ridge":
linear_model.Ridge(alpha=0.1, fit_intercept=False),
}
@pytest.fixture
def samples(joint):
return joint.sample(1000, rule="sobol")
@pytest.fixture
def evaluations(model_solver, samples):
return numpy.array([model_solver(sample) for sample in samples.T])
def fit_regression(P, x, u, rule="LS", retall=False, **kws):
"""
Fit a polynomial chaos expansion using linear regression.
Args:
P (Poly) : Polynomial expansion with `P.shape=(M,)` and `P.dim=D`.
x (array_like) : Collocation nodes with `x.shape=(D,K)`.
u (array_like) : Model evaluations with `len(u)=K`.
retall (bool) : If True return Fourier coefficients in addition to R.
rule (str) : Regression method used.
Returns:
(Poly, np.ndarray) : Fitted polynomial with `R.shape=u.shape[1:]` and
`R.dim=D`. The Fourier coefficients in the estimation.
Examples:
>>> x, y = cp.variable(2)
>>> P = cp.Poly([1, x, y])
>>> s = [[-1,-1,1,1], [-1,1,-1,1]]
>>> u = [0,1,1,2]
>>> print(cp.around(cp.fit_regression(P, s, u), 14))
0.5q0+0.5q1+1.0
"""
x = np.array(x)
if len(x.shape) == 1:
x = x.reshape(1, *x.shape)
u = np.array(u)
Q = P(*x).T
shape = u.shape[1:]
u = u.reshape(u.shape[0], int(np.prod(u.shape[1:])))
rule = rule.upper()
# Local rules
if rule == "LS":
uhat = linalg.lstsq(Q, u)[0].T
elif rule == "T":
uhat, alphas = rlstsq(Q, u, kws.get("order", 0),
kws.get("alpha", None), False, True)
uhat = uhat.T
elif rule == "TC":
uhat = rlstsq(Q, u, kws.get("order", 0), kws.get("alpha", None), True)
uhat = uhat.T
else:
# Scikit-learn wrapper
try:
_ = linear_model
except:
raise NotImplementedError("sklearn not installed")
if rule == "BARD":
solver = linear_model.ARDRegression(fit_intercept=False,
copy_X=False,
**kws)
elif rule == "BR":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.BayesianRidge(**kws)
elif rule == "EN":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.ElasticNet(**kws)
elif rule == "ENC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.ElasticNetCV(**kws)
elif rule == "LA": # success
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.Lars(**kws)
elif rule == "LAC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.LarsCV(**kws)
elif rule == "LAS":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.Lasso(**kws)
elif rule == "LASC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.LassoCV(**kws)
elif rule == "LL":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.LassoLars(**kws)
elif rule == "LLC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.LassoLarsCV(**kws)
elif rule == "LLIC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = linear_model.LassoLarsIC(**kws)
elif rule == "OMP":
solver = linear_model.OrthogonalMatchingPursuit(**kws)
uhat = solver.fit(Q, u).coef_
u = u.reshape(u.shape[0], *shape)
R = cp.poly.sum((P * uhat), -1)
R = cp.poly.reshape(R, shape)
if retall == 1:
return R, uhat
elif retall == 2:
if rule == "T":
return R, uhat, Q, alphas
return R, uhat, Q
return R
classification(svm.SVC(kernel="rbf", **SVC_PARAMS)),
classification(svm.NuSVC(kernel="rbf", **SVC_PARAMS)),
# Linear Regression
regression(linear_model.LinearRegression()),
regression(linear_model.HuberRegressor()),
regression(linear_model.ElasticNet(random_state=RANDOM_SEED)),
regression(linear_model.ElasticNetCV(random_state=RANDOM_SEED)),
regression(linear_model.TheilSenRegressor(random_state=RANDOM_SEED)),
regression(linear_model.Lars()),
regression(linear_model.LarsCV()),
regression(linear_model.Lasso(random_state=RANDOM_SEED)),
regression(linear_model.LassoCV(random_state=RANDOM_SEED)),
regression(linear_model.LassoLars()),
regression(linear_model.LassoLarsIC()),
regression(linear_model.OrthogonalMatchingPursuit()),
regression(linear_model.OrthogonalMatchingPursuitCV()),
regression(linear_model.Ridge(random_state=RANDOM_SEED)),
regression(linear_model.RidgeCV()),
regression(linear_model.BayesianRidge()),
regression(linear_model.ARDRegression()),
regression(linear_model.SGDRegressor(random_state=RANDOM_SEED)),
regression(
linear_model.PassiveAggressiveRegressor(random_state=RANDOM_SEED)),
# Logistic Regression
classification(
linear_model.LogisticRegression(random_state=RANDOM_SEED)),
classification(
linear_model.LogisticRegressionCV(random_state=RANDOM_SEED)),
classification(linear_model.RidgeClassifier(random_state=RANDOM_SEED)),
def train_sr(X_train, X_test, y_train, y_test, common_name_model, problemtype,
classes, default_features, transform_model, modeldir, settings):
# metrics
modeltypes = list()
explained_variances = list()
mean_absolute_errors = list()
mean_squared_errors = list()
median_absolute_errors = list()
r2_scores = list()
print(modeldir)
os.chdir(modeldir)
# make a temp folder to dump files into
foldername = ''
foldername = common_name_model + '_temp'
tempdir = os.getcwd() + '/' + foldername
try:
os.mkdir(foldername)
os.chdir(foldername)
except:
shutil.rmtree(foldername)
os.mkdir(foldername)
os.chdir(foldername)
# metrics.explained_variance_score(y_true, y_pred) Explained variance regression score function
# metrics.mean_absolute_error(y_true, y_pred) Mean absolute error regression loss
# metrics.mean_squared_error(y_true, y_pred[, …]) Mean squared error regression loss
# metrics.mean_squared_log_error(y_true, y_pred) Mean squared logarithmic error regression loss
# metrics.median_absolute_error(y_true, y_pred) Median absolute error regression loss
# metrics.r2_score(y_true, y_pred[, …]) R^2 (coefficient of determination) regression score function.
##################################################
## linear regression ##
##################################################
'''
LinearRegression fits a linear model with coefficients w = (w_1, ..., w_p)
to minimize the residual sum of squares between the observed responses
in the dataset, and the responses predicted by the linear approximation.
Example:
http://scikit-learn.org/stable/modules/linear_model.html
'''
try:
ols = linear_model.LinearRegression()
ols.fit(X_train, y_train)
#ols.predict(X_test, y_test)
predictions = cross_val_predict(ols, X_test, y_test, cv=6)
f = open('ols.pickle', 'wb')
pickle.dump(ols, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('linear regression')
except:
print('error - ORDINARY LEAST SQUARES')
##################################################
## Ridge regression ##
##################################################
'''
Ridge regression addresses some of the problems of
Ordinary Least Squares by imposing a penalty on the
size of coefficients.
The ridge coefficients minimize a penalized residual sum of squares.
Example:
http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge
'''
try:
ridge = linear_model.Ridge(fit_intercept=True,
alpha=0.0,
random_state=0,
normalize=True)
ridge.fit(X_train, y_train)
predictions = cross_val_predict(ridge, X_test, y_test, cv=6)
f = open('ridge.pickle', 'wb')
pickle.dump(ridge, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('ridge regression')
except:
print('error - RIDGE REGRESSION')
##################################################
## LASSO ##
##################################################
'''
The Lasso is a linear model that estimates sparse coefficients.
It is useful in some contexts due to its tendency to prefer solutions
with fewer parameter values, effectively reducing the number of
variables upon which the given solution is dependent.
For this reason, the Lasso and its variants are fundamental
to the field of compressed sensing. Under certain conditions,
it can recover the exact set of non-zero weights
(see Compressive sensing: tomography reconstruction with L1 prior (Lasso)).
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_lasso_model_selection.html#sphx-glr-auto-examples-linear-model-plot-lasso-model-selection-py
'''
try:
lasso = linear_model.Lasso(alpha=0.1)
lasso.fit(X_train, y_train)
predictions = cross_val_predict(lasso, X_test, y_test, cv=6)
f = open('lasso.pickle', 'wb')
pickle.dump(lasso, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('LASSO')
except:
print('error - LASSO')
##################################################
## Multi-task LASSO ##
##################################################
'''
The MultiTaskLasso is a linear model that estimates
sparse coefficients for multiple regression problems
jointly: y is a 2D array, of shape (n_samples, n_tasks).
The constraint is that the selected features are the same
for all the regression problems, also called tasks.
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_multi_task_lasso_support.html#sphx-glr-auto-examples-linear-model-plot-multi-task-lasso-support-py
'''
# # ONLY WORKS ON y_train that is multidimensional (one hot encoded)
# # Generate some 2D coefficients with sine waves with random frequency and phase
# mlasso = linear_model.MultiTaskLasso(alpha=0.1)
# mlasso.fit(X_train, y_train)
# predictions = cross_val_predict(mlasso, X_test, y_test, cv=6)
# accuracy = metrics.r2_score(y_test, predictions)
##################################################
## Elastic net ##
##################################################
'''
ElasticNet is a linear regression model trained with L1 and L2 prior as regularizer.
This combination allows for learning a sparse model where few of the weights are non-zero
like Lasso, while still maintaining the regularization properties of Ridge.
We control the convex combination of L1 and L2 using the l1_ratio parameter.
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_lasso_and_elasticnet.html#sphx-glr-auto-examples-linear-model-plot-lasso-and-elasticnet-py
'''
# need training data
try:
enet = linear_model.ElasticNet()
enet.fit(X_train, y_train)
predictions = cross_val_predict(enet, X_test, y_test, cv=6)
f = open('enet.pickle', 'wb')
pickle.dump(enet, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
ytest, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('elastic net')
except:
print('error - ELASTIC NET')
##################################################
## Multi-task elastic net ##
##################################################
'''
The MultiTaskElasticNet is an elastic-net model that estimates sparse coefficients
for multiple regression problems jointly: Y is a 2D array, of shape (n_samples, n_tasks).
The constraint is that the selected features are the same for all the regression problems,
also called tasks.
Example:
http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.MultiTaskElasticNet.html
'''
# # # ONLY WORKS ON y_train that is multidimensional (one hot encoded)
# clf = linear_model.MultiTaskElasticNet()
# clf.fit(X_train, y_train)
# #print(clf.coef_)
# #print(clf.intercept_)
##################################################
## Least angle regression (LARS) ##
##################################################
'''
The advantages of LARS are:
-> It is numerically efficient in contexts where p >> n (i.e., when the number of dimensions is significantly greater than the number of points)
-> It is computationally just as fast as forward selection and has the same order of complexity as an ordinary least squares.
-> It produces a full piecewise linear solution path, which is useful in cross-validation or similar attempts to tune the model.
-> If two variables are almost equally correlated with the response, then their coefficients should increase at approximately the same rate. The algorithm thus behaves as intuition would expect, and also is more stable.
-> It is easily modified to produce solutions for other estimators, like the Lasso.
The disadvantages of the LARS method include:
-> Because LARS is based upon an iterative refitting of the residuals,
-> it would appear to be especially sensitive to the effects of noise.
Example:
http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lars.html
'''
try:
lars = linear_model.Lars(n_nonzero_coefs=1)
lars.fit(X_train, y_train)
predictions = cross_val_predict(lars, X_test, y_test, cv=6)
f = open('lars.pickle', 'wb')
pickle.dump(lars, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('Least angle regression (LARS)')
except:
print('error - LARS')
##################################################
## LARS LASSO ##
##################################################
'''
LassoLars is a lasso model implemented using the LARS algorithm,
and unlike the implementation based on coordinate_descent,
this yields the exact solution, which is piecewise linear
as a function of the norm of its coefficients.
Example:
http://scikit-learn.org/stable/modules/linear_model.html#passive-aggressive-algorithms
'''
try:
lars_lasso = linear_model.LassoLars()
lars_lasso.fit(X_train, y_train)
predictions = cross_val_predict(lars_lasso, X_test, y_test, cv=6)
f = open('lars_lasso.pickle', 'wb')
pickle.dump(lars_lasso, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('LARS lasso')
except:
print('error - LARS LASSO')
##################################################
## Orthogonal Matching Pursuit (OMP) ##
##################################################
'''
OrthogonalMatchingPursuit and orthogonal_mp implements the OMP
algorithm for approximating the fit of a linear model with
constraints imposed on the number of non-zero coefficients (ie. the L 0 pseudo-norm).
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_omp.html#sphx-glr-auto-examples-linear-model-plot-omp-py
'''
try:
omp = linear_model.OrthogonalMatchingPursuit()
omp.fit(X_train, y_train)
predictions = cross_val_predict(omp, X_test, y_test, cv=6)
f = open('omp.pickle', 'wb')
pickle.dump(omp, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('orthogonal matching pursuit (OMP)')
except:
print('error - ORTHOGONAL MATCHING PURSUIT (OMP)')
##################################################
## Bayesian ridge regression ##
##################################################
'''
The advantages of Bayesian Regression are:
-> It adapts to the data at hand.
-> It can be used to include regularization parameters in the estimation procedure.
The disadvantages of Bayesian regression include:
-> Inference of the model can be time consuming.
Example:
http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.BayesianRidge.html
'''
# MULTI-DIMENSIONAL
# clf = BayesianRidge()
# clf.fit(X_train, y_train)
# predictions = cross_val_predict(clf, X_test, y_test, cv=6)
# accuracy = metrics.r2_score(y_test, predictions)
##################################################
## Automatic relevance determination ##
##################################################
'''
ARDRegression is very similar to Bayesian Ridge Regression,
but can lead to sparser weights w [1] [2]. ARDRegression poses
a different prior over w, by dropping the assumption of
the Gaussian being spherical.
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_ard.html#sphx-glr-auto-examples-linear-model-plot-ard-py
'''
# MULTI-DIMENSIONAL
# clf = ARDRegression(compute_score=True)
# clf.fit(X_train, y_train)
# predictions = cross_val_predict(clf, X_test, y_test, cv=6)
# accuracy = metrics.r2_score(y_test, predictions)
##################################################
## Logistic regression ##
##################################################
'''
Logistic regression, despite its name, is a linear model
for classification rather than regression. Logistic regression
is also known in the literature as logit regression,
maximum-entropy classification (MaxEnt) or the log-linear classifier.
In this model, the probabilities describing the possible outcomes
of a single trial are modeled using a logistic function.
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_logistic_l1_l2_sparsity.html#sphx-glr-auto-examples-linear-model-plot-logistic-l1-l2-sparsity-py
'''
try:
lr = linear_model.LogisticRegression(C=1.0, penalty='l1', tol=1e-6)
lr.fit(X_train, y_train)
predictions = cross_val_predict(lr, X_test, y_test, cv=6)
f = open('lr.pickle', 'wb')
pickle.dump(lr, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('logistic regression')
except:
print('error - LOGISTIC REGRESSION')
##################################################
## Stochastic gradient descent (SGD) ##
##################################################
'''
Stochastic gradient descent is a simple yet very efficient
approach to fit linear models. It is particularly useful
when the number of samples (and the number of features) is very large.
The partial_fit method allows only/out-of-core learning.
The classes SGDClassifier and SGDRegressor provide functionality
to fit linear models for classification and regression using
different (convex) loss functions and different penalties.
E.g., with loss="log", SGDClassifier fits a logistic regression model,
while with loss="hinge" it fits a linear support vector machine (SVM).
Example:
http://scikit-learn.org/stable/modules/sgd.html#sgd
'''
try:
# note you have to scale the data, as SGD algorithms are sensitive to
# feature scaling
scaler = StandardScaler()
scaler.fit(X_train)
X_train_2 = scaler.transform(X_train)
X_test_2 = scaler.transform(X_test)
sgd = linear_model.SGDRegressor()
sgd.fit(X_train_2, y_train)
predictions = cross_val_predict(sgd, X_test_2, y_test, cv=6)
f = open('sgd.pickle', 'wb')
pickle.dump(sgd, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('stochastic gradient descent (SGD)')
except:
print('error - STOCHASTIC GRADIENT DESCENT')
##################################################
## Perceptron algorithms ##
##################################################
'''
Multi-layer Perceptron is sensitive to feature scaling,
so it is highly recommended to scale your data.
For example, scale each attribute on the input vector X to [0, 1] or [-1, +1],
or standardize it to have mean 0 and variance 1.
Note that you must apply the same scaling to the test
set for meaningful results. You can use StandardScaler for standardization.
change the solver to 'lbfgs'. The default'adam' is a SGD-like method,
hich is effective for large & messy data but pretty useless for this kind of smooth & small data.
Example:
http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.PassiveAggressiveRegressor.html#sklearn.linear_model.PassiveAggressiveRegressor
'''
try:
nn = MLPRegressor(solver='lbfgs')
nn.fit(X_train, y_train)
predictions = cross_val_predict(nn, X_test, y_test, cv=6)
f = open('nn.pickle', 'wb')
pickle.dump(nn, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('perceptron')
except:
print('error - MLP REGRESSOR')
##################################################
## Passive-agressive algorithms ##
##################################################
'''
The passive-aggressive algorithms are a family of algorithms
for large-scale learning. They are similar to the Perceptron
in that they do not require a learning rate. However,
contrary to the Perceptron, they include a regularization parameter C.
Example:
http://jmlr.csail.mit.edu/papers/volume7/crammer06a/crammer06a.pdf
'''
try:
pa_regr = linear_model.PassiveAggressiveRegressor(random_state=0)
pa_regr.fit(X_train, y_train)
predictions = cross_val_predict(pa_regr, X_test, y_test, cv=6)
f = open('pa_regr.pickle', 'wb')
pickle.dump(pa_regr, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('passive-agressive algorithm')
except:
print('error - PASSIVE-AGGRESSIVE')
##################################################
## RANSAC ##
##################################################
'''
When in doubt, use RANSAC
RANSAC (RANdom SAmple Consensus) fits a model from random subsets of
inliers from the complete data set.
RANSAC is a non-deterministic algorithm producing only a reasonable
result with a certain probability, which is dependent on the number
of iterations (see max_trials parameter). It is typically used for
linear and non-linear regression problems and is especially popular
in the fields of photogrammetric computer vision.
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_ransac.html#sphx-glr-auto-examples-linear-model-plot-ransac-py
'''
try:
ransac = linear_model.RANSACRegressor()
ransac.fit(X_train, y_train)
predictions = cross_val_predict(ransac, X_test, y_test, cv=6)
f = open('ransac.pickle', 'wb')
pickle.dump(ransac, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('RANSAC')
except:
print('error - RANSAC')
##################################################
## Theil-SEN ##
##################################################
'''
The TheilSenRegressor estimator uses a generalization of the median
in multiple dimensions. It is thus robust to multivariate outliers.
Note however that the robustness of the estimator decreases quickly
with the dimensionality of the problem. It looses its robustness
properties and becomes no better than an ordinary least squares
in high dimension.
Note takes a bit longer to train.
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_theilsen.html#sphx-glr-auto-examples-linear-model-plot-theilsen-py
'''
try:
theilsen = linear_model.TheilSenRegressor(random_state=42)
theilsen.fit(X_train, y_train)
predictions = cross_val_predict(theilsen, X_test, y_test, cv=6)
f = open('theilsen.pickle', 'wb')
pickle.dump(theilsen, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('Theil-Sen')
except:
print('error - THEILSEN')
##################################################
## Huber Regression ##
##################################################
'''
The HuberRegressor is different to Ridge because it applies a linear loss
to samples that are classified as outliers. A sample is classified as an
inlier if the absolute error of that sample is lesser than a certain threshold.
It differs from TheilSenRegressor and RANSACRegressor because it does not
ignore the effect of the outliers but gives a lesser weight to them.
Example:
http://scikit-learn.org/stable/auto_examples/linear_model/plot_huber_vs_ridge.html#sphx-glr-auto-examples-linear-model-plot-huber-vs-ridge-py
'''
try:
huber = linear_model.HuberRegressor(fit_intercept=True,
alpha=0.0,
max_iter=100)
huber.fit(X_train, y_train)
predictions = cross_val_predict(huber, X_test, y_test, cv=6)
f = open('huber.pickle', 'wb')
pickle.dump(huber, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('huber regression')
except:
print('error - HUBER')
##################################################
## Polynomial Regression ##
##################################################
'''
One common pattern within machine learning is to use linear models trained on
nonlinear functions of the data. This approach maintains the generally fast
performance of linear methods, while allowing them to fit a much wider range of data.
Example:
http://scikit-learn.org/stable/modules/linear_model.html#passive-aggressive-algorithms
'''
try:
poly_lr = Pipeline([('poly',
PolynomialFeatures(degree=5, include_bias=False)),
('linreg', LinearRegression(normalize=True))])
poly_lr.fit(X_train, y_train)
predictions = cross_val_predict(poly_lr, X_test, y_test, cv=6)
accuracy = metrics.r2_score(y_test, predictions)
f = open('poly_lr.pickle', 'wb')
pickle.dump(poly_lr, f)
f.close()
# get stats
explained_variances, mean_absolute_errors, mean_squared_errors, median_absolute_errors, r2_scores = update_list(
y_test, predictions, explained_variances, mean_absolute_errors,
mean_squared_errors, median_absolute_errors, r2_scores)
modeltypes.append('polynomial (linear regression)')
except:
print('error - POLYNOMIAL')
##################################################
## Write session to .JSON ##
##################################################
os.chdir(modeldir)
print('\n\n')
print('RESULTS: \n')
# print table in terminal
table = BeautifulTable()
table.column_headers = ["model type", "R^2 score", "Mean Absolute Errors"]
print(len(modeltypes))
print(len(r2_scores))
print(len(mean_absolute_errors))
for i in range(len(modeltypes)):
table.append_row(
[modeltypes[i],
str(r2_scores[i]),
str(mean_absolute_errors[i])])
print(table)
filename = common_name_model + '.xlsx'
workbook = xlsxwriter.Workbook(filename)
worksheet = workbook.add_worksheet()
worksheet.write('A1', 'Model type')
worksheet.write('B1', 'R^2 score')
worksheet.write('C1', 'Explained Variances')
worksheet.write('D1', 'Mean Absolute Errors')
worksheet.write('E1', 'Mean Squared Log Errors')
worksheet.write('F1', 'Median Absolute Errors')
#worksheet.write('G1', 'Mean Squared Errors')
# print the best model in terms of mean abolute error
varnames = [
'ols.pickle', 'ridge.pickle', 'lasso.pickle', 'enet.pickle',
'lars.pickle', 'lars_lasso.pickle', 'omp.pickle', 'lr.pickle',
'sgd.pickle', 'nn.pickle', 'pa_regr.pickle', 'ransac.pickle',
'theilsen.pickle', 'huber.pickle', 'poly_lr.pickle'
]
# make sure all numbers, make mae 10 (a large number, to eliminate it from the list of options)
mae = mean_absolute_errors
for i in range(len(mae)):
if mae[i] == 'n/a':
mae[i] = 10
else:
mae[i] = float(mae[i])
# get minimim index and now delete temp folder, put master file in models directory
minval = np.amin(mae)
ind = mae.index(minval)
print('%s has the lowest mean absolute error (%s)' %
(modeltypes[ind], str(minval)))
# rename file
os.chdir(tempdir)
newname = common_name_model + '.pickle'
print('saving file to disk (%s)...' % (newname))
os.rename(varnames[ind], newname)
# move to models directory
shutil.copy(os.getcwd() + '/' + newname, modeldir + '/' + newname)
# now delete temp folder
os.chdir(modeldir)
shutil.rmtree(foldername)
# output spreadsheet of results and open up for analyis
for i in range(len(modeltypes)):
try:
worksheet.write('A' + str(i + 2), str(modeltypes[i]))
worksheet.write('B' + str(i + 2), str(r2_scores[i]))
worksheet.write('C' + str(i + 2), str(explained_variances[i]))
worksheet.write('D' + str(i + 2), str(mean_absolute_errors[i]))
worksheet.write('F' + str(i + 2), str(median_absolute_errors[i]))
#worksheet.write('G'+str(i+2), str(mean_squared_errors[i]))
except:
pass
workbook.close()
files = list()
files.append(common_name_model + '.xlsx')
files.append(common_name_model + '.pickle')
model_name = common_name_model + '.pickle'
model_dir = os.getcwd()
return model_name, model_dir, files
class EnsembleRegressor(BaseEstimator, MetaEstimatorMixin, RegressorMixin):
# Static member variables
_ensemble_regressors_auto = (
linear_model.LinearRegression(fit_intercept=True),
Pipeline([('poly', PolynomialFeatures(degree=2)),
('linear',
linear_model.LinearRegression(fit_intercept=False))]),
KernelRegression(kernel='poly'),
DecisionTreeRegressor(max_depth=4),
DecisionTreeRegressor(max_depth=None),
RandomForestRegressor(n_estimators=100),
)
_ensemble_possible_regressors = (
linear_model.LinearRegression(fit_intercept=True),
Pipeline([('poly', PolynomialFeatures(degree=2)),
('linear',
linear_model.LinearRegression(fit_intercept=False))]),
# # linear_model.Ridge(alpha=4, fit_intercept=True),
KernelRegression(kernel='poly'),
# linear_model.RidgeCV(alphas=[.01, .1, .3, .5, 1], fit_intercept=True),
# # linear_model.Lasso(alpha=4, fit_intercept=True),
# linear_model.LassoCV(n_alphas=100, fit_intercept=True, max_iter=5000),
# linear_model.ElasticNet(alpha=1),
# linear_model.ElasticNetCV(n_alphas=100, l1_ratio=.5),
# linear_model.OrthogonalMatchingPursuit(),
# linear_model.BayesianRidge(),
# # linear_model.ARDRegression(),
# linear_model.SGDRegressor(),
# # linear_model.PassiveAggressiveRegressor(loss='squared_epsilon_insensitive'),
# linear_model.RANSACRegressor(),
# LinearSVR(max_iter=1e4, fit_intercept=True, loss='squared_epsilon_insensitive', C=0.5),
# SVR(max_iter=1e4, kernel='poly', C=1, degree=4),
# SVR(max_iter=1e4, kernel='rbf', C=1, gamma=0.1),
# SVR(kernel='linear', C=1),
# SVR(kernel='linear', C=0.5),
# SVR(kernel='linear', C=0.1),
# DecisionTreeRegressor(max_depth=5),
DecisionTreeRegressor(max_depth=4),
DecisionTreeRegressor(max_depth=None),
RandomForestRegressor(n_estimators=100),
# AdaBoostRegressor(learning_rate=0.9, loss='square'),
# BaggingRegressor(),
MLPRegressor())
_ensemble_nn = [MLPRegressor(nb_epoch=1000) for _ in range(5)
] # 5 Multi Layer Perceptrons in the ensemble
_ensemble_nn_large = [MLPRegressor(nb_epoch=500) for _ in range(10)
] # 5 Multi Layer Perceptrons in the ensemble
_ensemble_ridge_regression = [
linear_model.Ridge(alpha=alpha, fit_intercept=True, normalize=True)
for alpha in np.arange(.1, 1, .2)
] # 5 Ridge Regressors
_ensemble_auto_large = (
linear_model.LinearRegression(fit_intercept=True),
Pipeline([('poly', PolynomialFeatures(degree=2)),
('linear',
linear_model.LinearRegression(fit_intercept=False))]),
linear_model.Ridge(alpha=0.5, fit_intercept=True, normalize=True),
KernelRegression(kernel='poly'),
# linear_model.RidgeCV(alphas=[.01, .1, .3, .5, 1], fit_intercept=True),
linear_model.Lasso(alpha=0.1, fit_intercept=True),
# linear_model.LassoCV(n_alphas=100, fit_intercept=True, max_iter=5000),
# linear_model.ElasticNet(alpha=1),
# linear_model.ElasticNetCV(n_alphas=100, l1_ratio=.5),
linear_model.OrthogonalMatchingPursuit(),
# linear_model.BayesianRidge(),
# # linear_model.ARDRegression(),
# linear_model.SGDRegressor(),
# linear_model.PassiveAggressiveRegressor(loss='squared_epsilon_insensitive'),
# linear_model.RANSACRegressor(),
LinearSVR(max_iter=1e3,
fit_intercept=True,
loss='squared_epsilon_insensitive',
C=1),
SVR(max_iter=1e3, kernel='poly', C=1, degree=3),
SVR(max_iter=1e3, kernel='rbf', C=1),
SVR(max_iter=1e3, kernel='sigmoid', C=1),
# SVR(kernel='linear', C=1),
# SVR(kernel='linear', C=0.5),
# SVR(kernel='linear', C=0.1),
# DecisionTreeRegressor(max_depth=5),
DecisionTreeRegressor(max_depth=4),
DecisionTreeRegressor(max_depth=None),
RandomForestRegressor(n_estimators=100),
AdaBoostRegressor(learning_rate=0.9, loss='square'),
BaggingRegressor(),
# MLPRegressor(num_hidden_units=5)
)
# self._ensemble_nn = [MLPRegressor(num_hidden_units=(i+6), nb_epoch=(i+6)*100) for i in range(5)] # 5 Multi Layer Perceptrons in the ensemble
def __init__(self, type='auto', verbose=False):
'''
:param type: Possible values: 'auto', 'mlp', 'mlp_large', 'ridge', 'auto_large' (defaults to 'auto').
Choice of set of regressors, 'auto' will use various standard regressors (usually linear
regression, NW-kernel, decision trees and random forests, but subject to change).
'mlp' will use 5 Multi-Layer Perceptrons, each with 10 hidden units, batch_size=32 and 1000 epochs.
'mlp_large' will use 10 MLPs, each with 10 hidden units, batch_size=32 and only 500 epochs.
'ridge' will train 5 ridge regressors with different alphas.
:param verbose:
'''
self._verbose = verbose
self.type = type.lower() # convert type to lowercase
if type == 'mlp':
self.regressors = EnsembleRegressor._ensemble_nn
elif type == 'mlp_large':
self.regressors = EnsembleRegressor._ensemble_nn_large
elif type == 'ridge':
self.regressors = EnsembleRegressor._ensemble_ridge_regression
elif type == 'auto_large':
self.regressors = EnsembleRegressor._ensemble_auto_large
else:
self.regressors = EnsembleRegressor._ensemble_regressors_auto
# set regressor labels
self.regressor_labels = []
self.regressor_count = len(self.regressors)
for i, regr in enumerate(self.regressors):
self.regressor_labels.append(str(regr))
def _dprint(self, *args, **kwargs):
"""overload print() function to only print when verbose=True."""
if self._verbose:
return __builtin__.print(*args, **kwargs)
def fit(self,
X_train,
y_train,
samples_per_regressor=None,
regressor_overlap=0):
""" Fits the model for all the regression algorithms in the ensemble.
The models themselves can be accessed directly at EnsembleRegressor.regressors,
and their labels is accessible in EnsembleRegressor.regressor_labels.
:param X_train: Data matrix. Shape [# samples, # features].
:param y_train: Target value vector.
:param samples_per_regressor: Number of samples from X_train that each regressor will be trained on.
Default 'None' will cause all regressors to be trained on all samples.
:param regressor_overlap: If samples_per_regressor is not None, this is the number of samples overlapping for
every adjacent pair of regressors. Defaults to no overlap.
"""
start_sample = 0
if samples_per_regressor is None:
end_sample = None
else:
end_sample = samples_per_regressor
start = time.time()
for i, regr in enumerate(self.regressors):
self._dprint('## ' + str(i) + '. ' + str(regr))
X = X_train[start_sample:end_sample, :]
y = y_train[start_sample:end_sample]
regr.fit(X, y)
if samples_per_regressor is not None:
start_sample = start_sample + samples_per_regressor - regressor_overlap
end_sample = start_sample + samples_per_regressor
if type(regr) in [
linear_model.LinearRegression, linear_model.Ridge,
LinearSVR
]:
self._dprint('\tCoefficients: ',
', '.join(['%.2f' % f for f in regr.coef_]))
if hasattr(regr, 'alphas_'):
self._dprint('\tAlphas: ',
', '.join(['%.2f' % f for f in regr.alphas_]))
self._dprint('Total running time: %.2f' % (time.time() - start))
def predict(self, X):
"""
:param X: Data matrix. Shape [# samples, # features].
:return: Ensemble predictions. Shape [# regressors, # samples].
"""
Z = np.ndarray(shape=(len(self.regressors), X.shape[0]))
for i, regr in enumerate(self.regressors):
# zip the real and predicted values together, sort them, and unzip them
try:
Z[i, :] = regr.predict(X)
except:
print(regr)
raise
return Z
def score(self, X_test, y_test, **kwargs):
"""
:return: vector with the R^2 score for each regressor
"""
s = np.zeros(self.regressor_count)
for i, regr in enumerate(self.regressors):
try:
s[i] = regr.score(X_test, y_test)
except:
print(regr)
raise
return s
def mean_squared_error(self, X_test, y_test):
"""
:return: vector with the MSE for each regressor
"""
Z = self.predict(X_test)
return np.mean((Z - y_test[None, :])**2, 1)
def get_regression_estimators(r, regression_models):
if r == 'ARDRegression':
regression_models[r] = linear_model.ARDRegression()
elif r == 'BayesianRidge':
regression_models[r] = linear_model.BayesianRidge()
elif r == 'ElasticNet':
regression_models[r] = linear_model.ElasticNet()
elif r == 'ElasticNetCV':
regression_models[r] = linear_model.ElasticNetCV()
elif r == 'HuberRegressor':
regression_models[r] = linear_model.HuberRegressor()
elif r == 'Lars':
regression_models[r] = linear_model.Lars()
elif r == 'LarsCV':
regression_models[r] = linear_model.LarsCV()
elif r == 'Lasso':
regression_models[r] = linear_model.Lasso()
elif r == 'LassoCV':
regression_models[r] = linear_model.LassoCV()
elif r == 'LassoLars':
regression_models[r] = linear_model.LassoLars()
elif r == 'LassoLarsCV':
regression_models[r] = linear_model.LassoLarsCV()
elif r == 'LassoLarsIC':
regression_models[r] = linear_model.LassoLarsIC()
elif r == 'LinearRegression':
regression_models[r] = linear_model.LinearRegression()
elif r == 'LogisticRegression':
regression_models[r] = linear_model.LogisticRegression()
elif r == 'LogisticRegressionCV':
regression_models[r] = linear_model.LogisticRegressionCV()
elif r == 'MultiTaskElasticNet':
regression_models[r] = linear_model.MultiTaskElasticNet()
elif r == 'MultiTaskElasticNetCV':
regression_models[r] = linear_model.MultiTaskElasticNetCV()
elif r == 'MultiTaskLasso':
regression_models[r] = linear_model.MultiTaskLasso()
elif r == 'MultiTaskLassoCV':
regression_models[r] = linear_model.MultiTaskLassoCV()
elif r == 'OrthogonalMatchingPursuit':
regression_models[r] = linear_model.OrthogonalMatchingPursuit()
elif r == 'OrthogonalMatchingPursuitCV':
regression_models[r] = linear_model.OrthogonalMatchingPursuitCV()
elif r == 'PassiveAggressiveClassifier':
regression_models[r] = linear_model.PassiveAggressiveClassifier()
elif r == 'PassiveAggressiveRegressor':
regression_models[r] = linear_model.PassiveAggressiveRegressor()
elif r == 'Perceptron':
regression_models[r] = linear_model.Perceptron()
elif r == 'RANSACRegressor':
regression_models[r] = linear_model.RANSACRegressor()
elif r == 'Ridge':
regression_models[r] = linear_model.Ridge()
elif r == 'RidgeClassifier':
regression_models[r] = linear_model.RidgeClassifier()
elif r == 'RidgeClassifierCV':
regression_models[r] = linear_model.RidgeClassifierCV()
elif r == 'RidgeCV':
regression_models[r] = linear_model.RidgeCV()
elif r == 'SGDClassifier':
regression_models[r] = linear_model.SGDClassifier()
elif r == 'SGDRegressor':
regression_models[r] = linear_model.SGDRegressor()
elif r == 'TheilSenRegressor':
regression_models[r] = linear_model.TheilSenRegressor()
else:
print(
r +
" is an unsupported regression type. Check if you have misspelled the name."
)
def generate_prediction(cls, race):
"""Generate a prediction for the specified race"""
prediction = {
'race_id': race['_id'],
'earliest_date': cls.get_earliest_date(),
'prediction_version': cls.PREDICTION_VERSION,
'seed_version': Seed.SEED_VERSION,
'results': None,
'score': None,
'train_seeds': None,
'test_seeds': None,
'estimator': None
}
predictor = None
generate_predictor = False
segment = tuple(race['entry_conditions']) + tuple(
[race['track_condition']])
with cls.predictor_cache_lock:
if segment in cls.predictor_cache:
predictor = cls.predictor_cache[segment]
else:
cls.predictor_cache[segment] = None
generate_predictor = True
if generate_predictor:
similar_races = pyracing.Race.find({
'entry_conditions':
race['entry_conditions'],
'track_condition':
race['track_condition'],
'start_time': {
'$lt': race.meet['date']
}
})
if len(similar_races) >= (1 / cls.TEST_SIZE):
try:
train_races, test_races = cross_validation.train_test_split(
similar_races, test_size=cls.TEST_SIZE)
train_X = []
train_y = []
for train_race in train_races:
for seed in train_race.seeds:
if seed['result'] is not None:
train_X.append(seed.normalized_data)
train_y.append(seed['result'])
test_X = []
test_y = []
for test_race in test_races:
for seed in test_race.seeds:
if seed['result'] is not None:
test_X.append(seed.normalized_data)
test_y.append(seed['result'])
predictor = {
'classifier': None,
'score': None,
'train_seeds': len(train_y),
'test_seeds': len(test_y),
'estimator': None
}
dual = len(train_X) < len(train_X[0])
kernel = 'linear'
loss = 'epsilon_insensitive'
if not dual:
loss = 'squared_epsilon_insensitive'
for estimator in (
linear_model.BayesianRidge(),
linear_model.ElasticNet(),
linear_model.LinearRegression(),
linear_model.LogisticRegression(),
linear_model.OrthogonalMatchingPursuit(),
linear_model.PassiveAggressiveRegressor(),
linear_model.Perceptron(), linear_model.Ridge(),
linear_model.SGDRegressor(),
svm.SVR(kernel=kernel),
svm.LinearSVR(dual=dual,
loss=loss), svm.NuSVR(kernel=kernel),
tree.DecisionTreeRegressor(),
tree.ExtraTreeRegressor()):
logging.debug(
'Trying {estimator} for {segment}'.format(
estimator=estimator.__class__.__name__,
segment=segment))
try:
classifier = pipeline.Pipeline([
('feature_selection',
feature_selection.SelectFromModel(
estimator, 'mean')),
('regression', estimator)
])
classifier.fit(train_X, train_y)
score = classifier.score(test_X, test_y)
if predictor['classifier'] is None or predictor[
'score'] is None or score > predictor[
'score']:
logging.debug(
'Using {estimator} ({score}) for {segment}'
.format(
estimator=estimator.__class__.__name__,
score=score,
segment=segment))
predictor['classifier'] = classifier
predictor['score'] = score
predictor[
'estimator'] = estimator.__class__.__name__
except BaseException as e:
logging.debug(
'Caught exception while trying {estimator} for {segment}: {exception}'
.format(estimator=estimator.__class__.__name__,
segment=segment,
exception=e))
continue
cls.predictor_cache[segment] = predictor
except:
del cls.predictor_cache[segment]
raise
else:
del cls.predictor_cache[segment]
else:
while predictor is None:
try:
predictor = cls.predictor_cache[segment]
time.sleep(10)
except KeyError:
break
if predictor is not None:
reverse = False
if 'score' in predictor and predictor['score'] is not None:
reverse = predictor['score'] < 0
prediction['score'] = abs(predictor['score'])
if 'classifier' in predictor and predictor[
'classifier'] is not None:
raw_results = {}
for seed in race.seeds:
raw_result = predictor['classifier'].predict(
numpy.array(seed.normalized_data).reshape(1, -1))[0]
if raw_result is not None:
if not raw_result in raw_results:
raw_results[raw_result] = []
raw_results[raw_result].append(seed.runner['number'])
for key in sorted(raw_results.keys(), reverse=reverse):
if prediction['results'] is None:
prediction['results'] = []
prediction['results'].append(
sorted([number for number in raw_results[key]]))
if 'train_seeds' in predictor:
prediction['train_seeds'] = predictor['train_seeds']
if 'test_seeds' in predictor:
prediction['test_seeds'] = predictor['test_seeds']
if 'estimator' in predictor:
prediction['estimator'] = predictor['estimator']
return prediction
import pickle
url = "https://goo.gl/sXleFv"
names = [
'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX',
'PTRATIO', 'B', 'LSTAT', 'MEDV'
]
dataframe = read_csv(url, delim_whitespace=True, names=names)
array = dataframe.values
X = array[:, 0:13]
Y = array[:, 13]
kfold = KFold(n_splits=10, random_state=7)
linregression = lm.LinearRegression()
ridge = lm.Ridge()
lasso = lm.Lasso()
lars = lm.Lars()
omp = lm.OrthogonalMatchingPursuit()
br = lm.BayesianRidge()
kn = KNeighborsRegressor()
svr = svm.SVR()
dtr = tree.DecisionTreeRegressor()
rfr = ensemble.RandomForestRegressor()
gbr = ensemble.GradientBoostingRegressor()
bag = ensemble.BaggingRegressor(br)
mse = 'neg_mean_squared_error'
r2 = 'r2'
models = [
linregression, ridge, lasso, lars, omp, br, kn, svr, dtr, rfr, gbr, bag
]
for model in models:
mseResult = cross_val_score(model, X, Y, cv=kfold, scoring=mse)
r2result = cross_val_score(model, X, Y, cv=kfold, scoring=r2)
def train_test_all_regressors(X_train, X_test, y_train, y_test, seed=SEED):
"""
Train, test and print the results of most available regressors presented in sklearn.
Args:
X_train (matrix): matrix with features of the training set
y_train (list): list of values of target of the training set
X_test (matrix): matrix with features of the test set
y_test (list): list of values of target of the test set
"""
assert isinstance(X_train, pd.core.frame.DataFrame)
assert isinstance(X_test, pd.core.frame.DataFrame)
assert isinstance(y_train, pd.core.series.Series)
assert isinstance(y_test, pd.core.series.Series)
assert isinstance(seed, int)
from sklearn import linear_model
from sklearn import tree
from sklearn import ensemble
from sklearn import neighbors
from sklearn import neural_network
models = []
models.append(("BayesianRidge", linear_model.BayesianRidge()))
models.append(("ElasticNet", linear_model.ElasticNet()))
models.append(("HuberRegressor", linear_model.HuberRegressor()))
models.append(("Lars", linear_model.Lars()))
models.append(("Lasso", linear_model.Lasso()))
models.append(("LassoLars", linear_model.LassoLars()))
models.append(("LinearRegression", linear_model.LinearRegression()))
models.append(("OrthogonalMatchingPursuit",
linear_model.OrthogonalMatchingPursuit()))
models.append(("PassiveAggressiveRegressor",
linear_model.PassiveAggressiveRegressor()))
models.append(("Ridge", linear_model.Ridge()))
models.append(("SGDRegressor", linear_model.SGDRegressor()))
models.append(
("AdaBoostRegressor", ensemble.AdaBoostRegressor(random_state=seed)))
models.append(
("BaggingRegressor", ensemble.BaggingRegressor(random_state=seed)))
models.append(("ExtraTreesRegressor",
ensemble.ExtraTreesRegressor(random_state=seed)))
models.append(("GradientBoostingRegressor",
ensemble.GradientBoostingRegressor(random_state=seed)))
models.append(("RandomForestRegressor",
ensemble.RandomForestRegressor(random_state=seed)))
models.append(("DecisionTreeRegressor",
tree.DecisionTreeRegressor(random_state=seed)))
models.append(("KNeighborsRegressor", neighbors.KNeighborsRegressor()))
models.append(("MLPRegressor", neural_network.MLPRegressor()))
best_mean_absolute_percentage_error = 100
best_model = ''
for name, model in models:
print(
'------------------------------------------------------------------------------'
)
print(name)
print(
'------------------------------------------------------------------------------'
)
model.fit(X_train, y_train)
print('Training Set')
y_pred = model.predict(X_train)
print_results(y_train, y_pred)
print('Testing Set')
y_pred = model.predict(X_test)
print_results(y_test, y_pred)
mean_absolute_percentage_error_value = mean_absolute_percentage_error(
y_test, y_pred)
if mean_absolute_percentage_error_value < best_mean_absolute_percentage_error:
best_mean_absolute_percentage_error = mean_absolute_percentage_error
best_model = name
print(
'------------------------------------------------------------------------------'
)
print('Best model: ' + best_model)
print('Best mean absolute percentage error: ' +
str(best_mean_absolute_percentage_error))
print(
'------------------------------------------------------------------------------'
)
def __init__(self, method, yrange, params, i=0): #TODO: yrange doesn't currently do anything. Remove or do something with it!
self.algorithm_list = ['PLS',
'GP',
'OLS',
'OMP',
'Lasso',
'Elastic Net',
'Ridge',
'Bayesian Ridge',
'ARD',
'LARS',
'LASSO LARS',
'SVR',
'KRR',
]
self.method = method
self.outliers = None
self.ransac = False
print(params)
if self.method[i] == 'PLS':
self.model = PLSRegression(**params[i])
if self.method[i] == 'OLS':
self.model = linear.LinearRegression(**params[i])
if self.method[i] == 'OMP':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
self.model = linear.OrthogonalMatchingPursuit(**params_temp)
if self.method[i] == 'LASSO':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
self.model = linear.Lasso(**params_temp)
if self.method[i] == 'Elastic Net':
params_temp = copy.copy(params[i])
self.model = linear.ElasticNet(**params_temp)
if self.method[i] == 'Ridge':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
self.model = linear.Ridge(**params_temp)
if self.method[i] == 'BRR':
self.model = linear.BayesianRidge(**params[i])
if self.method[i] == 'ARD':
self.model = linear.ARDRegression(**params[i])
if self.method[i] == 'LARS':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
self.model = linear.Lars(**params_temp)
if self.method[i] == 'LASSO LARS':
self.model = linear.LassoLars(**params)
if self.method[i] == 'SVR':
self.model = svm.SVR(**params[i])
if self.method[i] == 'KRR':
self.model = kernel_ridge.KernelRidge(**params[i])
if self.method[i] == 'GP':
# get the method for dimensionality reduction and the number of components
self.reduce_dim = params[i]['reduce_dim']
self.n_components = params[i]['n_components']
# create a temporary set of parameters
params_temp = copy.copy(params[i])
# Remove parameters not accepted by Gaussian Process
params_temp.pop('reduce_dim')
params_temp.pop('n_components')
self.model = GaussianProcess(**params_temp)
]
extract_operators = gen_extract_operators(ancestor_size, downsampled_size,
patchsize, ancestor_shift)
image = scipy.misc.imread('./lena.png')
image = numpy.array(image) / 255.
y = gen_patch_2d(image, patchsize, data_shift)
# y = y[:, numpy.random.choice(y.shape[1], 3000)]
y_mean = numpy.mean(y, axis=0)
y = y - numpy.tile(y_mean, [y.shape[0], 1])
# declare lasso model
lasso = linear_model.Lasso(alpha=1e-3)
# omp = linear_model.OrthogonalMatchingPursuit(tol=0.1, normalize=False)
omp = linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=15,
normalize=False)
# aal = AncestralAtomLearning(ancestor, extract_operators, omp)
# remember datetime for filename
dtstr = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
init_ancestors = []
for i in range(10):
theta = numpy.linspace(0, 2 * numpy.pi, ancestor_size[0])
sin_wave = numpy.sin((i + 1) * theta)
ancestor_init = numpy.outer(sin_wave, sin_wave)
init_ancestors.append(ancestor_init.flatten('F'))
init_ancestors = numpy.array(init_ancestors).T
for num_ancestor in range(1, 10):
def __init__(
self,
method,
yrange,
params,
i=0
): #TODO: yrange doesn't currently do anything. Remove or do something with it!
self.algorithm_list = [
'PLS',
'GP',
'OLS',
'OMP',
'Lasso',
'Elastic Net',
'Ridge',
'Bayesian Ridge',
'ARD',
'LARS',
'LASSO LARS',
'SVR',
'KRR',
]
self.method = method
self.outliers = None
self.ransac = False
print(params)
if self.method[i] == 'PLS':
self.model = PLSRegression(**params[i])
if self.method[i] == 'OLS':
self.model = linear.LinearRegression(**params[i])
if self.method[i] == 'OMP':
# check whether to do CV or not
self.do_cv = params[i]['CV']
# create a temporary set of parameters
params_temp = copy.copy(params[i])
# Remove CV parameter
params_temp.pop('CV')
if self.do_cv is False:
self.model = linear.OrthogonalMatchingPursuit(**params_temp)
else:
params_temp.pop('precompute')
self.model = linear.OrthogonalMatchingPursuitCV(**params_temp)
if self.method[i] == 'LASSO':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
# check whether to do CV or not
try:
self.do_cv = params[i]['CV']
# Remove CV parameter
params_temp.pop('CV')
except:
self.do_cv = False
if self.do_cv is False:
self.model = linear.Lasso(**params_temp)
else:
params_temp.pop('alpha')
self.model = linear.LassoCV(**params_temp)
if self.method[i] == 'Elastic Net':
params_temp = copy.copy(params[i])
try:
self.do_cv = params[i]['CV']
params_temp.pop('CV')
except:
self.do_cv = False
if self.do_cv is False:
self.model = linear.ElasticNet(**params_temp)
else:
params_temp['l1_ratio'] = [.1, .5, .7, .9, .95, .99, 1]
self.model = linear.ElasticNetCV(**params_temp)
if self.method[i] == 'Ridge':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
try:
# check whether to do CV or not
self.do_cv = params[i]['CV']
# Remove CV parameter
params_temp.pop('CV')
except:
self.do_cv = False
if self.do_cv:
self.model = linear.RidgeCV(**params_temp)
else:
self.model = linear.Ridge(**params_temp)
if self.method[i] == 'BRR':
self.model = linear.BayesianRidge(**params[i])
if self.method[i] == 'ARD':
self.model = linear.ARDRegression(**params[i])
if self.method[i] == 'LARS':
# create a temporary set of parameters
params_temp = copy.copy(params[i])
try:
# check whether to do CV or not
self.do_cv = params[i]['CV']
# Remove CV parameter
params_temp.pop('CV')
except:
self.do_cv = False
if self.do_cv is False:
self.model = linear.Lars(**params_temp)
else:
self.model = linear.LarsCV(**params_temp)
if self.method[i] == 'LASSO LARS':
model = params[i]['model']
params_temp = copy.copy(params[i])
params_temp.pop('model')
if model == 0:
self.model = linear.LassoLars(**params_temp)
elif model == 1:
self.model = linear.LassoLarsCV(**params_temp)
elif model == 2:
self.model = linear.LassoLarsIC(**params_temp)
else:
print("Something went wrong, \'model\' should be 0, 1, or 2")
if self.method[i] == 'SVR':
self.model = svm.SVR(**params[i])
if self.method[i] == 'KRR':
self.model = kernel_ridge.KernelRidge(**params[i])
if self.method[i] == 'GP':
# get the method for dimensionality reduction and the number of components
self.reduce_dim = params[i]['reduce_dim']
self.n_components = params[i]['n_components']
# create a temporary set of parameters
params_temp = copy.copy(params[i])
# Remove parameters not accepted by Gaussian Process
params_temp.pop('reduce_dim')
params_temp.pop('n_components')
self.model = GaussianProcess(**params_temp)
def fit_regression(P, x, u, rule="LS", retall=False, **kws):
"""
Fit a polynomial chaos expansion using linear regression.
Parameters
----------
P : Poly
Polynomial chaos expansion with `P.shape=(M,)` and `P.dim=D`.
x : array_like
Collocation nodes with `x.shape=(D,K)`.
u : array_like
Model evaluations with `len(u)=K`.
retall : bool
If True return uhat in addition to R
rule : str
Regression method used.
The follwong methods uses scikits-learn as backend.
See `sklearn.linear_model` for more details.
Key Scikit-learn Description
--- ------------ -----------
Parameters Description
---------- -----------
"BARD" ARDRegression Bayesian ARD Regression
n_iter=300 Maximum iterations
tol=1e-3 Optimization tolerance
alpha_1=1e-6 Gamma scale parameter
alpha_2=1e-6 Gamma inverse scale parameter
lambda_1=1e-6 Gamma shape parameter
lambda_2=1e-6 Gamma inverse scale parameter
threshold_lambda=1e-4 Upper pruning threshold
"BR" BayesianRidge Bayesian Ridge Regression
n_iter=300 Maximum iterations
tol=1e-3 Optimization tolerance
alpha_1=1e-6 Gamma scale parameter
alpha_2=1e-6 Gamma inverse scale parameter
lambda_1=1e-6 Gamma shape parameter
lambda_2=1e-6 Gamma inverse scale parameter
"EN" ElastiNet Elastic Net
alpha=1.0 Dampening parameter
rho Mixing parameter in [0,1]
max_iter=300 Maximum iterations
tol Optimization tolerance
"ENC" ElasticNetCV EN w/Cross Validation
rho Dampening parameter(s)
eps=1e-3 min(alpha)/max(alpha)
n_alphas Number of alphas
alphas List of alphas
max_iter Maximum iterations
tol Optimization tolerance
cv=3 Cross validation folds
"LA" Lars Least Angle Regression
n_nonzero_coefs Number of non-zero coefficients
eps Cholesky regularization
"LAC" LarsCV LAR w/Cross Validation
max_iter Maximum iterations
cv=5 Cross validation folds
max_n_alphas Max points for residuals in cv
"LAS" Lasso Least Absolute Shrinkage and
Selection Operator
alpha=1.0 Dampening parameter
max_iter Maximum iterations
tol Optimization tolerance
"LASC" LassoCV LAS w/Cross Validation
eps=1e-3 min(alpha)/max(alpha)
n_alphas Number of alphas
alphas List of alphas
max_iter Maximum iterations
tol Optimization tolerance
cv=3 Cross validation folds
"LL" LassoLars Lasso and Lars model
max_iter Maximum iterations
eps Cholesky regularization
"LLC" LassoLarsCV LL w/Cross Validation
max_iter Maximum iterations
cv=5 Cross validation folds
max_n_alphas Max points for residuals in cv
eps Cholesky regularization
"LLIC" LassoLarsIC LL w/AIC or BIC
criterion "AIC" or "BIC" criterion
max_iter Maximum iterations
eps Cholesky regularization
"OMP" OrthogonalMatchingPursuit
n_nonzero_coefs Number of non-zero coefficients
tol Max residual norm (instead of non-zero coef)
Local methods
Key Description
--- -----------
"LS" Ordenary Least Squares
"T" Ridge Regression/Tikhonov Regularization
order Order of regularization (or custom matrix)
alpha Dampning parameter (else estimated from gcv)
"TC" T w/Cross Validation
order Order of regularization (or custom matrix)
alpha Dampning parameter (else estimated from gcv)
Returns
-------
R[, uhat]
R : Poly
Fitted polynomial with `R.shape=u.shape[1:]` and `R.dim=D`.
uhat : np.ndarray
The Fourier coefficients in the estimation.
Examples
--------
>>> P = cp.Poly([1, x, y])
>>> s = [[-1,-1,1,1], [-1,1,-1,1]]
>>> u = [0,1,1,2]
>>> print fit_regression(P, s, u)
0.5q1+0.5q0+1.0
"""
x = np.array(x)
if len(x.shape) == 1:
x = x.reshape(1, *x.shape)
u = np.array(u)
Q = P(*x).T
shape = u.shape[1:]
u = u.reshape(u.shape[0], int(np.prod(u.shape[1:])))
rule = rule.upper()
# Local rules
if rule == "LS":
uhat = la.lstsq(Q, u)[0].T
elif rule == "T":
uhat, alphas = rlstsq(Q, u, kws.get("order", 0),
kws.get("alpha", None), False, True)
uhat = uhat.T
elif rule == "TC":
uhat = rlstsq(Q, u, kws.get("order", 0), kws.get("alpha", None), True)
uhat = uhat.T
else:
# Scikit-learn wrapper
try:
_ = lm
except:
raise NotImplementedError("sklearn not installed")
if rule == "BARD":
solver = lm.ARDRegression(fit_intercept=False, copy_X=False, **kws)
elif rule == "BR":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.BayesianRidge(**kws)
elif rule == "EN":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.ElasticNet(**kws)
elif rule == "ENC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.ElasticNetCV(**kws)
elif rule == "LA": # success
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.Lars(**kws)
elif rule == "LAC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.LarsCV(**kws)
elif rule == "LAS":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.Lasso(**kws)
elif rule == "LASC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.LassoCV(**kws)
elif rule == "LL":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.LassoLars(**kws)
elif rule == "LLC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.LassoLarsCV(**kws)
elif rule == "LLIC":
kws["fit_intercept"] = kws.get("fit_intercept", False)
solver = lm.LassoLarsIC(**kws)
elif rule == "OMP":
solver = lm.OrthogonalMatchingPursuit(**kws)
uhat = solver.fit(Q, u).coef_
u = u.reshape(u.shape[0], *shape)
R = po.sum((P * uhat), -1)
R = po.reshape(R, shape)
if retall == 1:
return R, uhat
elif retall == 2:
if rule == "T":
return R, uhat, Q, alphas
return R, uhat, Q
return R
# Prepare ensemble regressors
regressors = (
linear_model.LinearRegressi on(fit_intercept=True),
Pipeline(
[('poly', PolynomialFeatures(degree=2)),
('linear', linear_model.LinearRegression(fit_intercept=False))]
),
linear_model.Ridge(alpha=.1, fit_intercept=True),
linear_model.RidgeCV(alphas=[.01, .1, .3, .5, 1], fit_intercept=True),
linear_model.Lasso(alpha=1, fit_intercept=True),
linear_model.LassoCV(n_alphas=100, fit_intercept=True),
linear_model.ElasticNet(alpha=1),
linear_model.ElasticNetCV(n_alphas=100, l1_ratio=.5),
linear_model.OrthogonalMatchingPursuit(),
linear_model.BayesianRidge(),
linear_model.ARDRegression(),
linear_model.SGDRegressor(),
linear_model.PassiveAggressiveRegressor(loss='squared_epsilon_insensitive'),
linear_model.RANSACRegressor(),
LinearSVR(max_iter=1e4, fit_intercept=True, loss='squared_epsilon_insensitive', C=0.5),
SVR(max_iter=1e4, kernel='poly', C=1, degree=4),
SVR(max_iter=1e4, kernel='rbf', C=1, gamma=0.1),
SVR(kernel='linear', C=1),
SVR(kernel='linear', C=0.5),
SVR(kernel='linear', C=0.1),
DecisionTreeRegressor(max_depth=5),
DecisionTreeRegressor(max_depth=4),
DecisionTreeRegressor(max_depth=None),
RandomForestRegressor(n_estimators=100),
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn import linear_model
classifiers = {
"Nearest Neighbors" : KNeighborsClassifier(3),
"LinearRegression": linear_model.LinearRegression(),
"Ridge": linear_model.Ridge(alpha = .5),
"Lasso": linear_model.Lasso(alpha = 0.1),
"ElasticNet": linear_model.ElasticNet(random_state=0),
"Lars": linear_model.Lars(n_nonzero_coefs=1),
"LassoLars": linear_model.LassoLars(alpha=.1),
"Omp": linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=1),
"BayesianRidge":linear_model.BayesianRidge(),
"ARDRegression":linear_model.ARDRegression(),
"LogisitcRegression":linear_model.LogisticRegression(),
"SGDClassifier":linear_model.SGDClassifier(),
"Perceptron": linear_model.Perceptron(),
"PassiveAggressiveClassifier": linear_model.PassiveAggressiveClassifier(),
"Theil-Sen": linear_model.TheilSenRegressor(random_state=42),
"RANSAC": linear_model.RANSACRegressor(random_state=42),
"Huber": linear_model.HuberRegressor(),
"SVC linear": SVC(kernel="linear", C=0.025),
"SVC": SVC(gamma=2, C=1, probability=True),
"GuassianProcess":GaussianProcessClassifier(1.0 * RBF(1.0)),
"DecisionTree":DecisionTreeClassifier(max_depth=5),
"RandomForest":RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
"NeutraNet":MLPClassifier(alpha=1),
def train_test_all_regressors_with_cross_validation(X, y, seed=SEED):
"""
Train, test and print the results of most available regressors presented in sklearn using cross validation.
Args:
X_train (matrix): matrix with features of the training set
y_train (list): list of values of target of the training set
X_test (matrix): matrix with features of the test set
y_test (list): list of values of target of the test set
"""
assert isinstance(X, pd.core.frame.DataFrame)
assert isinstance(y, pd.core.series.Series)
assert isinstance(seed, int)
from sklearn import linear_model
from sklearn import tree
from sklearn import ensemble
from sklearn import neighbors
from sklearn import neural_network
from sklearn.model_selection import cross_val_score
models = []
models.append(("BayesianRidge", linear_model.BayesianRidge()))
models.append(("ElasticNet", linear_model.ElasticNet()))
models.append(("HuberRegressor", linear_model.HuberRegressor()))
models.append(("Lars", linear_model.Lars()))
models.append(("Lasso", linear_model.Lasso()))
models.append(("LassoLars", linear_model.LassoLars()))
models.append(("LinearRegression", linear_model.LinearRegression()))
models.append(("OrthogonalMatchingPursuit",
linear_model.OrthogonalMatchingPursuit()))
models.append(("PassiveAggressiveRegressor",
linear_model.PassiveAggressiveRegressor()))
models.append(("Ridge", linear_model.Ridge()))
models.append(("SGDRegressor", linear_model.SGDRegressor()))
models.append(
("AdaBoostRegressor", ensemble.AdaBoostRegressor(random_state=seed)))
models.append(
("BaggingRegressor", ensemble.BaggingRegressor(random_state=seed)))
models.append(("ExtraTreesRegressor",
ensemble.ExtraTreesRegressor(random_state=seed)))
models.append(("GradientBoostingRegressor",
ensemble.GradientBoostingRegressor(random_state=seed)))
models.append(("RandomForestRegressor",
ensemble.RandomForestRegressor(random_state=seed)))
models.append(("DecisionTreeRegressor",
tree.DecisionTreeRegressor(random_state=seed)))
models.append(("KNeighborsRegressor", neighbors.KNeighborsRegressor()))
models.append(("MLPRegressor", neural_network.MLPRegressor()))
best_rmse = 1000000000.0
best_model = ''
for name, model in models:
print(
'------------------------------------------------------------------------------'
)
print(name)
print(
'------------------------------------------------------------------------------'
)
scores = cross_val_score(model,
X,
y,
scoring='neg_root_mean_squared_error',
cv=5)
scores = -scores
scores_mean = scores.mean()
scores_std = scores.std()
print("RMSE: %0.3f (+/- %0.2f)" % (scores_mean, scores_std * 2))
#mean_absolute_percentage_error_value = mean_absolute_percentage_error(y_test, y_pred)
if scores_mean < best_rmse:
best_rmse = scores_mean
best_model = name
print(
'------------------------------------------------------------------------------'
)
print('Best model: ' + best_model)
print('Best RMSE: ' + str(best_rmse))
print(
'------------------------------------------------------------------------------'
)
import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model
from sklearn.metrics import r2_score
from sklearn.datasets import make_sparse_coded_signal
if __name__ == "__main__":
print("Generating data...")
ncomp, nf, nncoef = 256, 10000, 32
y, X, w = make_sparse_coded_signal(n_samples=1,
n_components=ncomp,
n_features=nf,
n_nonzero_coefs=nncoef)
idx, = w.nonzero()
y = y + 0.02 * np.random.randn(len(y))
y = y.flatten()
X_train, X_test = X[nf // 2:], X[:nf // 2]
y_train, y_test = y[nf // 2:], y[:nf // 2]
print(X, y)
print("Fitting model...")
omp = linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=nncoef)
omp.fit(X_train, y_train)
print("R2 score: {0}".format(r2_score(y_test, omp.predict(X_test))))
plt.scatter(np.arange(nf // 2), y_train, color="purple")
plt.scatter(np.arange(nf // 2) + (nf // 2), y_test, color="red")
plt.plot(np.arange(nf // 2), omp.predict(X_train), color="purple")
plt.plot(np.arange(nf // 2) + (nf // 2), omp.predict(X_test), color="red")
plt.show()
