def run(argv=None):
"""Emulate a HP search and monitor fit time."""
args = parser.parse_args(argv)
imputers = {
'Mean': SimpleImputer(strategy='mean'),
'Mean+mask': SimpleImputer(strategy='mean', add_indicator=True),
'Med': SimpleImputer(strategy='median'),
'Med+mask': SimpleImputer(strategy='median', add_indicator=True),
'Iterative': IterativeImputer(max_iter=args.max_iter),
'Iterative+mask': IterativeImputer(add_indicator=True,
max_iter=args.max_iter),
'IterativeR': IterativeImputer(estimator=RidgeCV(),
max_iter=args.max_iter),
'IterativeR+mask': IterativeImputer(estimator=RidgeCV(),
add_indicator=True,
max_iter=args.max_iter),
'KNN': KNNImputer(),
'KNN+mask': KNNImputer(add_indicator=True),
}
task_name = args.task_name
est = args.est
imp = imputers.get(args.imp, None)
if task_name is None or est is None:
logger.info('No argv given.')
task_name = 'TB/shock_hemo'
est = 'HGBC'
task = tasks[task_name]
logger.info(f'Argv given. Task {task.meta.tag}. est {est}.')
t0 = time()
logger.info('Getting X.')
X = task.X
logger.info('Getting y.')
y = task.y
logger.info(f'X shape before splits: {X.shape}')
# Simulate the outer CV (the one of KFold)
X_train, _, y_train, _ = train_test_split(X, y, test_size=0.2)
# Simulate the inner CV (the one of RandomSearchCV)
X_train2, X_test2, y_train2, _ = train_test_split(X_train, y_train, test_size=0.2)
# Now X has the same shape as in real experiment
logger.info(f'X shape: {X_train2.shape}')
t_X_ready = time()
if imp is not None:
logger.info(f'Fitting imputer {args.imp}')
imp.fit(X_train2, y_train2)
t_fit_imp = time()
logger.info('Imputer fitted.')
logger.info('Transforming X_train')
imp.transform(X_train2)
t_tra1_imp = time()
logger.info('X_train transformed')
logger.info('Transforming X_test')
imp.transform(X_test2)
t_tra2_imp = time()
logger.info('X_test transformed')
t_fits = [time()]
for learning_rate in param_space['learning_rate']:
for max_depth in param_space['max_depth']:
if est == 'HGBC':
estimator = HistGradientBoostingClassifier(
learning_rate=learning_rate,
max_depth=max_depth
)
elif est == 'HGBR':
estimator = HistGradientBoostingRegressor(
loss='least_absolute_deviation',
learning_rate=learning_rate,
max_depth=max_depth
)
else:
raise ValueError(f'Unknown estimator {est}')
logger.info(f'Params: LR {learning_rate} MD {max_depth}')
logger.info('Fitting estimator.')
estimator.fit(X_train2, y_train2)
t_fits.append(time())
logger.info('Estimator fitted.')
t_fits = np.diff(t_fits)
data = {
'task_tag': [task.meta.tag],
'imp': [args.imp],
'imp_params': [repr({'max_iter': args.max_iter})],
'X_shape': [repr(X.shape)],
'X_train_shape': [repr(X_train2.shape)],
'X_test_shape': [repr(X_test2.shape)],
'time_X_ready': [t_X_ready-t0],
'time_fit_imp': np.around([0 if imp is None else t_fit_imp-t_X_ready], 2),
'time_tra1_imp': np.around([0 if imp is None else t_tra1_imp-t_X_ready], 2),
'time_tra2_imp': np.around([0 if imp is None else t_tra2_imp-t_tra1_imp], 2),
'time_fits': [repr(np.around(t_fits.tolist(), 2))],
'time_fits_mean': [np.around(t_fits.mean(), 2)]
}
new_df = pd.DataFrame(data)
df = None
filepath = 'results/fit_time.csv'
if os.path.exists(filepath):
df = pd.read_csv(filepath, index_col=0)
if df is not None:
new_df = pd.concat([df, new_df])
new_df.to_csv(filepath)
def test_same_predictions_regression(seed, min_samples_leaf, n_samples,
max_leaf_nodes):
# Make sure sklearn has the same predictions as lightgbm for easy targets.
#
# In particular when the size of the trees are bound and the number of
# samples is large enough, the structure of the prediction trees found by
# LightGBM and sklearn should be exactly identical.
#
# Notes:
# - Several candidate splits may have equal gains when the number of
# samples in a node is low (and because of float errors). Therefore the
# predictions on the test set might differ if the structure of the tree
# is not exactly the same. To avoid this issue we only compare the
# predictions on the test set when the number of samples is large enough
# and max_leaf_nodes is low enough.
# - To ignore discrepancies caused by small differences the binning
# strategy, data is pre-binned if n_samples > 255.
# - We don't check the least_absolute_deviation loss here. This is because
# LightGBM's computation of the median (used for the initial value of
# raw_prediction) is a bit off (they'll e.g. return midpoints when there
# is no need to.). Since these tests only run 1 iteration, the
# discrepancy between the initial values leads to biggish differences in
# the predictions. These differences are much smaller with more
# iterations.
pytest.importorskip("lightgbm")
rng = np.random.RandomState(seed=seed)
max_iter = 1
max_bins = 255
X, y = make_regression(n_samples=n_samples,
n_features=5,
n_informative=5,
random_state=0)
if n_samples > 255:
# bin data and convert it to float32 so that the estimator doesn't
# treat it as pre-binned
X = _BinMapper(n_bins=max_bins + 1).fit_transform(X).astype(np.float32)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng)
est_sklearn = HistGradientBoostingRegressor(
max_iter=max_iter,
max_bins=max_bins,
learning_rate=1,
early_stopping=False,
min_samples_leaf=min_samples_leaf,
max_leaf_nodes=max_leaf_nodes)
est_lightgbm = get_equivalent_estimator(est_sklearn, lib='lightgbm')
est_lightgbm.fit(X_train, y_train)
est_sklearn.fit(X_train, y_train)
# We need X to be treated an numerical data, not pre-binned data.
X_train, X_test = X_train.astype(np.float32), X_test.astype(np.float32)
pred_lightgbm = est_lightgbm.predict(X_train)
pred_sklearn = est_sklearn.predict(X_train)
# less than 1% of the predictions are different up to the 3rd decimal
assert np.mean(abs(pred_lightgbm - pred_sklearn) > 1e-3) < .011
if max_leaf_nodes < 10 and n_samples >= 1000:
pred_lightgbm = est_lightgbm.predict(X_test)
pred_sklearn = est_sklearn.predict(X_test)
# less than 1% of the predictions are different up to the 4th decimal
assert np.mean(abs(pred_lightgbm - pred_sklearn) > 1e-4) < .01
# class even more optimized for large dataset called
# `HistGradientBoostingClassifier` and `HistGradientBoostingRegressor`. Each
# feature in the dataset `X` is first binned by computing histograms which are
# later used to evaluate the potential splits. The number of splits to evaluate
# is then much smaller. This algorithm becomes much more efficient than
# gradient boosting when the dataset has 10,000+ samples.
#
# Below we will give an example of a large dataset and we can compare
# computation time with the earlier experiment in the previous section.
# %%
from time import time
from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingRegressor
histogram_gradient_boosting = HistGradientBoostingRegressor(max_iter=200,
random_state=0)
start_time = time()
histogram_gradient_boosting.fit(X_train, y_train)
fit_time_histogram_gradient_boosting = time() - start_time
start_time = time()
score_histogram_gradient_boosting = histogram_gradient_boosting.score(
X_test, y_test)
score_time_histogram_gradient_boosting = time() - start_time
print("Historgram gradient boosting decision tree")
print(f"R2 score: {score_histogram_gradient_boosting:.3f}")
print(f"Fit time: {fit_time_histogram_gradient_boosting:.2f} s")
print(f"Score time: {score_time_histogram_gradient_boosting:.5f} s\n")
def test_poisson_y_positive(y):
# Test that ValueError is raised if either one y_i < 0 or sum(y_i) <= 0.
err_msg = r"loss='poisson' requires non-negative y and sum\(y\) > 0."
gbdt = HistGradientBoostingRegressor(loss="poisson", random_state=0)
with pytest.raises(ValueError, match=err_msg):
gbdt.fit(np.zeros(shape=(len(y), 1)), y)
pipe_KN = Pipeline([('scl', StandardScaler()),
('est', KNeighborsRegressor())])
#pipe_KN_PCA = Pipeline([('scl', StandardScaler()),
# ('pca', PCA(n_components=2)),
# ('est', KNeighborsRegressor())])
pipe_GB = Pipeline([('scl', StandardScaler()),
('est', GradientBoostingRegressor(random_state=seed))])
#pipe_GB_PCA = Pipeline([('scl', StandardScaler()),
# ('pca', PCA(n_components=2)),
# ('est', GradientBoostingRegressor(random_state=seed))])
pipe_HGB = Pipeline([('scl', StandardScaler()),
('est', HistGradientBoostingRegressor(random_state=seed))])
#pipe_HGB_PCA = Pipeline([('scl', StandardScaler()),
# ('pca', PCA(n_components=2)),
# ('est', HistGradientBoostingRegressor(random_state=seed))])
pipe_B = Pipeline([('scl', StandardScaler()),
('est', BaggingRegressor(random_state=seed))])
#pipe_B_PCA = Pipeline([('scl', StandardScaler()),
# ('pca', PCA(n_components=2)),
# ('est', BaggingRegressor(random_state=seed))])
# Set grid search params
grid_params_DT = [{'est__criterion': ('mse', 'friedman_mse', 'mae'),
'est__splitter': ('best', 'random'),
'for the California housing dataset, with MLPRegressor')
display.figure_.subplots_adjust(hspace=0.3)
# %%
# Gradient boosting
# .................
#
# Let's now fit a :class:`~sklearn.ensemble.HistGradientBoostingRegressor` and
# compute the partial dependence on the same features.
from sklearn.experimental import enable_hist_gradient_boosting # noqa
from sklearn.ensemble import HistGradientBoostingRegressor
print("Training HistGradientBoostingRegressor...")
tic = time()
est = HistGradientBoostingRegressor()
est.fit(X_train, y_train)
print(f"done in {time() - tic:.3f}s")
print(f"Test R2 score: {est.score(X_test, y_test):.2f}")
# %%
# Here, we used the default hyperparameters for the gradient boosting model
# without any preprocessing as tree-based models are naturally robust to
# monotonic transformations of numerical features.
#
# Note that on this tabular dataset, Gradient Boosting Machines are both
# significantly faster to train and more accurate than neural networks. It is
# also significantly cheaper to tune their hyperparameters (the defaults tend
# to work well while this is not often the case for neural networks).
#
# We will plot the partial dependence, both individual (ICE) and averaged one
# the prediction performance of each individual model.
#
# Here, we combine 3 learners (linear and non-linear) and use a ridge regressor
# to combine their outputs together.
from sklearn.ensemble import StackingRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.experimental import enable_hist_gradient_boosting # noqa
from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn.linear_model import LassoCV
from sklearn.linear_model import RidgeCV
estimators = [
('Random Forest', RandomForestRegressor(random_state=42)),
('Lasso', LassoCV()),
('Gradient Boosting', HistGradientBoostingRegressor(random_state=0))
]
stacking_regressor = StackingRegressor(
estimators=estimators, final_estimator=RidgeCV()
)
###############################################################################
# We used the Boston data set (prediction of house prices). We check the
# performance of each individual predictor as well as the stack of the
# regressors.
import time
import numpy as np
from sklearn.datasets import load_boston
from sklearn.model_selection import cross_validate, cross_val_predict
# 'poisson' loss as well. import numpy as np from sklearn.model_selection import train_test_split from sklearn.linear_model import PoissonRegressor from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingRegressor n_samples, n_features = 1000, 20 rng = np.random.RandomState(0) X = rng.randn(n_samples, n_features) # positive integer target correlated with X[:, 5] with many zeros: y = rng.poisson(lam=np.exp(X[:, 5]) / 2) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) glm = PoissonRegressor() gbdt = HistGradientBoostingRegressor(loss='poisson', learning_rate=.01) glm.fit(X_train, y_train) gbdt.fit(X_train, y_train) print(glm.score(X_test, y_test)) print(gbdt.score(X_test, y_test)) ############################################################################## # Rich visual representation of estimators # ----------------------------------------- # Estimators can now be visualized in notebooks by enabling the # `display='diagram'` option. This is particularly useful to summarise the # structure of pipelines and other composite estimators, with interactivity to # provide detail. Click on the example image below to expand Pipeline # elements. See :ref:`visualizing_composite_estimators` for how you can use # this feature.
# df_test = pd.read_csv("./data/test_3B.csv")
print('train:', df_train.shape)
print('test:', df_test.shape)
# extract the whole features for ML
X = df_train[df_train.columns[:-1]].copy()
y = df_train["SalePrice_Log"].copy()
final_test = df_test.copy() # final test X
#%% Models with optimal params determined by 10-folds CV
lasso_opt = Lasso(alpha=0.00423)
ridge_opt = Ridge(alpha=1)
knn_opt = KNeighborsRegressor(n_neighbors=10, weights='distance')
rf = RandomForestRegressor(random_state=2) # use default params: N=100
boost = HistGradientBoostingRegressor() # default 'max_iter': 100
dt = DecisionTreeRegressor(random_state=5) # use default params
SVMrbf_opt = SVR(kernel='rbf', C=1.0, gamma=0.01)
SVMli = SVR(kernel='linear')
SVMpoly_opt = SVR(kernel='poly', degree=3)
estimators_list = [('Lasso', lasso_opt), ('Ridge', ridge_opt),
('KNN', knn_opt), ('RF', rf), ('DT', dt), ('Boost', boost),
('SVMrbf', SVMrbf_opt), ('SVMli', SVMli),
('SVMpoly', SVMpoly_opt)]
#%% do SFS-model stacking
start_time = time.time()
estimators_sel, RMSE_best, RMSE_all_steps = SFS_stack_models(estimators_list,
X,
y,
"Partial dependence of house value on non-location features\n"
"for the California housing dataset, with MLPRegressor")
display.figure_.subplots_adjust(hspace=0.3)
# %%
# Gradient boosting
# .................
#
# Let's now fit a :class:`~sklearn.ensemble.HistGradientBoostingRegressor` and
# compute the partial dependence on the same features.
from sklearn.ensemble import HistGradientBoostingRegressor
print("Training HistGradientBoostingRegressor...")
tic = time()
est = HistGradientBoostingRegressor(random_state=0)
est.fit(X_train, y_train)
print(f"done in {time() - tic:.3f}s")
print(f"Test R2 score: {est.score(X_test, y_test):.2f}")
# %%
# Here, we used the default hyperparameters for the gradient boosting model
# without any preprocessing as tree-based models are naturally robust to
# monotonic transformations of numerical features.
#
# Note that on this tabular dataset, Gradient Boosting Machines are both
# significantly faster to train and more accurate than neural networks. It is
# also significantly cheaper to tune their hyperparameters (the defaults tend
# to work well while this is not often the case for neural networks).
#
# We will plot the partial dependence, both individual (ICE) and averaged one
# Gradient boosting estimator with dropped categorical features
# -------------------------------------------------------------
# As a baseline, we create an estimator where the categorical features are
# dropped:
from sklearn.experimental import enable_hist_gradient_boosting # noqa
from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn.pipeline import make_pipeline
from sklearn.compose import make_column_transformer
from sklearn.compose import make_column_selector
dropper = make_column_transformer(
('drop', make_column_selector(dtype_include='category')),
remainder='passthrough')
hist_dropped = make_pipeline(dropper,
HistGradientBoostingRegressor(random_state=42))
# %%
# Gradient boosting estimator with one-hot encoding
# -------------------------------------------------
# Next, we create a pipeline that will one-hot encode the categorical features
# and let the rest of the numerical data to passthrough:
from sklearn.preprocessing import OneHotEncoder
one_hot_encoder = make_column_transformer(
(OneHotEncoder(sparse=False, handle_unknown='ignore'),
make_column_selector(dtype_include='category')),
remainder='passthrough')
hist_one_hot = make_pipeline(one_hot_encoder,
categories = [
["clear", "misty", "rain"],
["spring", "summer", "fall", "winter"],
["False", "True"],
["False", "True"],
]
ordinal_encoder = OrdinalEncoder(categories=categories)
gbrt_pipeline = make_pipeline(
ColumnTransformer(
transformers=[
("categorical", ordinal_encoder, categorical_columns),
],
remainder="passthrough",
),
HistGradientBoostingRegressor(categorical_features=range(4), ),
)
# %%
#
# Lets evaluate our gradient boosting model with the mean absolute error of the
# relative demand averaged accross our 5 time-based cross-validation splits:
def evaluate(model, X, y, cv):
cv_results = cross_validate(
model,
X,
y,
cv=ts_cv,
scoring=["neg_mean_absolute_error", "neg_root_mean_squared_error"],
"""Hydro_Model
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1aMtPXxewSC8pS3Wp1Ay7Z8kthRyY-Ko6
"""
import pandas as pd
import numpy as np
from sklearn.experimental import enable_hist_gradient_boosting # noqa
from sklearn.ensemble import HistGradientBoostingRegressor
import pickle
data = pd.read_csv('clean.csv')
X = data.drop(['Unnamed: 0', 'ID', 'Date', 'WQI', 'Label'], axis=1)
Y = data['WQI']
X = np.array(X)
Y = np.array(Y)
model = HistGradientBoostingRegressor().fit(X, Y)
model.predict(x)
pkl_filename = "pickle_model.pkl"
with open(pkl_filename, 'wb') as file:
pickle.dump(model, file)
print("Model Trained and Saved")
#RMSE ~ 540 ($)
#RUNTIME ~ 20 (min)
'RandomForestRegressor':RandomForestRegressor(),
#RMSE ~ 551 ($)
#RUNTIME ~ 4 (min)
======================================== Experimental Models ========================================
"""
highest_score = 530
ESTIMATORS_STACK = [
# to be used with StackingRegressor() estimator
('RandomForestRegressor', RandomForestRegressor()),
('HistGradientBoostingRegressor', HistGradientBoostingRegressor())
]
models = {
'RandomForestRegressor': RandomForestRegressor(),
#RMSE ~ 551 ($)
#RUNTIME ~ 4 (min)
#'StackingRegressor':StackingRegressor(estimators=ESTIMATORS_STACK),
#COMMENT: Can stack multiple estimators and generate a regression from it.
#RMSE ~ 528 ($)
#RUNTIME ~ 30 (min)
#'ARDRegression':ARDRegression(),
#RMSE ~ UNKOWN needs more RAM
#RUNTIME ~ UNKOWN (min)
from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn.preprocessing import OrdinalEncoder
tree_preprocessor = ColumnTransformer(
[
("categorical", OrdinalEncoder(),
["VehBrand", "VehPower", "VehGas", "Region", "Area"]),
("numeric", "passthrough",
["VehAge", "DrivAge", "BonusMalus", "Density"]),
],
remainder="drop",
)
poisson_gbrt = Pipeline([
("preprocessor", tree_preprocessor),
("regressor", HistGradientBoostingRegressor(loss="poisson",
max_leaf_nodes=128)),
])
poisson_gbrt.fit(df_train, df_train["Frequency"],
regressor__sample_weight=df_train["Exposure"])
print("Poisson Gradient Boosted Trees evaluation:")
score_estimator(poisson_gbrt, df_test)
# %%
# Like the Poisson GLM above, the gradient boosted trees model minimizes
# the Poisson deviance. However, because of a higher predictive power,
# it reaches lower values of Poisson deviance.
#
# Evaluating models with a single train / test split is prone to random
# fluctuations. If computing resources allow, it should be verified that
# cross-validated performance metrics would lead to similar conclusions.
# top_5_precip = pd.concat([top_5_flow, p_data], axis=1, join='inner').drop('66401_0', axis = 1).dropna()
# top_5_train_labels = np.array(top_5['66401_0'])
# top_5_train_features = np.array(top_5.drop('66401_0', axis = 1))
# top_5_test_labels = np.array(top_5_flow[:date_cut_off])
# top_5_test_features = np.array(top_5_precip.loc[:date_cut_off])
# top_5_test_features_index = top_5_precip.loc[:date_cut_off].index
test_features = data1.loc[date_cut_off:]
test_features1 = np.array(test_features.drop(['66401_0', '66401_48'], axis=1))
test_features2 = np.array(test_features.drop('66401_0', axis=1))
rf = HistGradientBoostingRegressor(max_iter=100, random_state=42)
rf.fit(train_features1, train_labels)
predictions1 = rf.predict(test_features1)
predict1 = pd.Series(predictions1,
index=test_features.index,
name='100 HistGB Predicted Flow')
# median1 = actual1[actual1 > actual1.quantile(0.95)].median()
# predict3 = predict2[predict2 > median1]
# rf = HistGradientBoostingRegressor(max_iter = 200, random_state = 42)
# rf.fit(train_features1, train_labels)
# predictions2 = rf.predict(test_features1)
# predict2 = pd.Series(predictions2, index=test_features.index, name='200 HistGB Predicted Flow')
# rf = RandomForestRegressor(n_estimators = 200, random_state = None, n_jobs=3)
# rf.fit(top_5_train_features, top_5_train_labels)
"AdaBoost_Regressor",
"LogisticRegression",
"TheilSen_Regressor",
"Huber_Regressor",
"CatBoost_Classifier",
"CatBoost_Regressor",
]
classifiers = [
RidgeClassifier(),
RidgeClassifierCV(),
XGBRegressor(),
GradientBoostingClassifier(verbose=0),
GradientBoostingRegressor(verbose=0),
HistGradientBoostingClassifier(verbose=0),
HistGradientBoostingRegressor(verbose=0),
ExtraTreesClassifier(verbose=0),
ExtraTreesRegressor(verbose=0),
LinearDiscriminantAnalysis(),
QuadraticDiscriminantAnalysis(),
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5, n_estimators=500, verbose=0),
RandomForestRegressor(max_depth=5, n_estimators=500, verbose=0),
AdaBoostClassifier(**PARAMS_ADABOOST),
AdaBoostRegressor(**PARAMS_ADABOOST),
LogisticRegression(max_iter=10000, verbose=0),
TheilSenRegressor(verbose=False),
HuberRegressor(),
CatBoostClassifier(**PARAMS_CATBOOST),
CatBoostRegressor(**PARAMS_CATBOOST_REGRESSOR),
]
print(' %7s %17.4f %15.4f ' %
('MedAE:', round(medae_in, 2), round(medae_out, 2)))
print(' %7s %17.4f %15.4f ' %
('MSLE:', round(msle_in, 2), round(msle_out, 2)))
print(' %7s %17.4f %15.4f ' % ('rmspe', rmspe_in, rmspe_out))
#----------------------------------------------------------------------
# Treinar e testar um regressor HistGradientBoosting
#----------------------------------------------------------------------
print(' ')
print(' REGRESSOR HIST GRADIENT BOOSTING:')
print(' ')
hgb = HistGradientBoostingRegressor(l2_regularization=12.0,
max_iter=70,
learning_rate=0.1,
loss='least_absolute_deviation')
hgb = hgb.fit(x_treino, y_treino)
y_resposta_treino = hgb.predict(x_treino)
y_resposta_teste = hgb.predict(x_teste)
print(' Métrica DENTRO da amostra FORA da amostra ')
print(' ------- ----------------- --------------- ')
mse_in = mean_squared_error(y_treino, y_resposta_treino)
rmse_in = math.sqrt(mse_in)
r2_in = r2_score(y_treino, y_resposta_treino)
medae_in = median_absolute_error(y_treino, y_resposta_treino)
msle_in = mean_squared_log_error(y_treino, y_resposta_treino)
def create_estimators(
preprocessing: ColumnTransformer) -> Dict[str, GridSearchCV]:
"""Create estimators for model fitting.
ElasticNet, HistGradientBoost and RandomForest regressors are added.
More can be added.
Args:
preprocessing: Preprocessing Pipeline
Returns:
Key-value pairs of estimator name and instantiated estimator
"""
# ElasticNet (alpha = 1.0 -> Lasso)
param_grid = {
"regressor__regressor__alpha": (0.001, 0.01, 0.1, 1.0),
"regressor__regressor__l1_ratio": (0.05, 0.2, 0.5, 0.7, 0.9, 1.0),
}
en_pipe = TransformedTargetRegressor(
regressor=Pipeline([("preprocessor", preprocessing),
("regressor", ElasticNet())]),
transformer=StandardScaler(),
)
en_search = GridSearchCV(
en_pipe,
param_grid=param_grid,
cv=5,
)
# RandomForest
param_grid = {
"regressor__regressor__n_estimators": [50, 100, 200],
"regressor__regressor__max_depth": [5, 6, 7, 15],
}
rf_pipe = TransformedTargetRegressor(
regressor=Pipeline([("preprocessor", preprocessing),
("regressor", RandomForestRegressor())]),
transformer=StandardScaler(),
)
rf_search = GridSearchCV(
rf_pipe,
param_grid=param_grid,
cv=5,
)
# HistGradientBoost
param_grid = {
"regressor__regressor__l2_regularization": [0.0, 0.1, 1.0],
"regressor__regressor__max_depth": [6, 15],
"regressor__regressor__max_iter": [100, 200],
}
hgb_pipe = TransformedTargetRegressor(
regressor=Pipeline([
("preprocessor", preprocessing),
("regressor", HistGradientBoostingRegressor()),
]),
transformer=StandardScaler(),
)
hgb_search = GridSearchCV(
hgb_pipe,
param_grid=param_grid,
cv=5,
)
return {
"ElasticNet": en_search,
"RandomForest": rf_search,
"HistGradientBoost": hgb_search,
}
import matplotlib.pyplot as plt
rng = np.random.RandomState(0)
n_samples = 5000
f_0 = rng.rand(n_samples) # positive correlation with y
f_1 = rng.rand(n_samples) # negative correlation with y
X = np.c_[f_0, f_1]
noise = rng.normal(loc=0.0, scale=0.01, size=n_samples)
y = 5 * f_0 + np.sin(10 * np.pi * f_0) - 5 * f_1 - np.cos(
10 * np.pi * f_1) + noise
fig, ax = plt.subplots()
# Without any constraint
gbdt = HistGradientBoostingRegressor()
gbdt.fit(X, y)
disp = PartialDependenceDisplay.from_estimator(
gbdt,
X,
features=[0, 1],
line_kw={
"linewidth": 4,
"label": "unconstrained",
"color": "tab:blue"
},
ax=ax,
)
# With positive and negative constraints
gbdt = HistGradientBoostingRegressor(monotonic_cst=[1, -1])
def test_absolute_error():
# For coverage only.
X, y = make_regression(n_samples=500, random_state=0)
gbdt = HistGradientBoostingRegressor(loss="absolute_error", random_state=0)
gbdt.fit(X, y)
assert gbdt.score(X, y) > 0.9
# preprocessed output from the 3 learners.
from sklearn.experimental import enable_hist_gradient_boosting # noqa
from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import StackingRegressor
from sklearn.linear_model import LassoCV
from sklearn.linear_model import RidgeCV
lasso_pipeline = make_pipeline(processor_lin, LassoCV())
rf_pipeline = make_pipeline(processor_nlin,
RandomForestRegressor(random_state=42))
gradient_pipeline = make_pipeline(
processor_nlin, HistGradientBoostingRegressor(random_state=0))
estimators = [('Random Forest', rf_pipeline), ('Lasso', lasso_pipeline),
('Gradient Boosting', gradient_pipeline)]
stacking_regressor = StackingRegressor(estimators=estimators,
final_estimator=RidgeCV())
# %%
# Measure and plot the results
##############################################################################
#
# Now we can use Ames Housing dataset to make the predictions. We check the
# performance of each individual predictor as well as of the stack of the
# regressors.
#
def test_missing_values_minmax_imputation():
# Compare the buit-in missing value handling of Histogram GBC with an
# a-priori missing value imputation strategy that should yield the same
# results in terms of decision function.
#
# Each feature (containing NaNs) is replaced by 2 features:
# - one where the nans are replaced by min(feature) - 1
# - one where the nans are replaced by max(feature) + 1
# A split where nans go to the left has an equivalent split in the
# first (min) feature, and a split where nans go to the right has an
# equivalent split in the second (max) feature.
#
# Assuming the data is such that there is never a tie to select the best
# feature to split on during training, the learned decision trees should be
# strictly equivalent (learn a sequence of splits that encode the same
# decision function).
#
# The MinMaxImputer transformer is meant to be a toy implementation of the
# "Missing In Attributes" (MIA) missing value handling for decision trees
# https://www.sciencedirect.com/science/article/abs/pii/S0167865508000305
# The implementation of MIA as an imputation transformer was suggested by
# "Remark 3" in https://arxiv.org/abs/1902.06931
class MinMaxImputer(TransformerMixin, BaseEstimator):
def fit(self, X, y=None):
mm = MinMaxScaler().fit(X)
self.data_min_ = mm.data_min_
self.data_max_ = mm.data_max_
return self
def transform(self, X):
X_min, X_max = X.copy(), X.copy()
for feature_idx in range(X.shape[1]):
nan_mask = np.isnan(X[:, feature_idx])
X_min[nan_mask, feature_idx] = self.data_min_[feature_idx] - 1
X_max[nan_mask, feature_idx] = self.data_max_[feature_idx] + 1
return np.concatenate([X_min, X_max], axis=1)
def make_missing_value_data(n_samples=int(1e4), seed=0):
rng = np.random.RandomState(seed)
X, y = make_regression(n_samples=n_samples,
n_features=4,
random_state=rng)
# Pre-bin the data to ensure a deterministic handling by the 2
# strategies and also make it easier to insert np.nan in a structured
# way:
X = KBinsDiscretizer(n_bins=42, encode="ordinal").fit_transform(X)
# First feature has missing values completely at random:
rnd_mask = rng.rand(X.shape[0]) > 0.9
X[rnd_mask, 0] = np.nan
# Second and third features have missing values for extreme values
# (censoring missingness):
low_mask = X[:, 1] == 0
X[low_mask, 1] = np.nan
high_mask = X[:, 2] == X[:, 2].max()
X[high_mask, 2] = np.nan
# Make the last feature nan pattern very informative:
y_max = np.percentile(y, 70)
y_max_mask = y >= y_max
y[y_max_mask] = y_max
X[y_max_mask, 3] = np.nan
# Check that there is at least one missing value in each feature:
for feature_idx in range(X.shape[1]):
assert any(np.isnan(X[:, feature_idx]))
# Let's use a test set to check that the learned decision function is
# the same as evaluated on unseen data. Otherwise it could just be the
# case that we find two independent ways to overfit the training set.
return train_test_split(X, y, random_state=rng)
# n_samples need to be large enough to minimize the likelihood of having
# several candidate splits with the same gain value in a given tree.
X_train, X_test, y_train, y_test = make_missing_value_data(
n_samples=int(1e4), seed=0)
# Use a small number of leaf nodes and iterations so as to keep
# under-fitting models to minimize the likelihood of ties when training the
# model.
gbm1 = HistGradientBoostingRegressor(max_iter=100,
max_leaf_nodes=5,
random_state=0)
gbm1.fit(X_train, y_train)
gbm2 = make_pipeline(MinMaxImputer(), clone(gbm1))
gbm2.fit(X_train, y_train)
# Check that the model reach the same score:
assert gbm1.score(X_train,
y_train) == pytest.approx(gbm2.score(X_train, y_train))
assert gbm1.score(X_test,
y_test) == pytest.approx(gbm2.score(X_test, y_test))
# Check the individual prediction match as a finer grained
# decision function check.
assert_allclose(gbm1.predict(X_train), gbm2.predict(X_train))
assert_allclose(gbm1.predict(X_test), gbm2.predict(X_test))
# parameters = {
# 'learning_rate':[0.05, 0.10, 0.15, 0.20],
# 'max_depth':[3, 4, 5, 6, 7, 8, 9],
# 'l2_regularization':[0.0, 0.0001, 0.001, 0.01, 0.1]
# }
# model = HistGradientBoostingClassifier(max_iter=300)
# GDSCV = GridSearchCV(estimator=model, param_grid=parameters, cv=5, scoring='r2', n_jobs=-1)
# GDSCV.fit(X_train, y_train)
# print(GDSCV.best_params_)
# cv_score = cross_val_score(model, X_train, y_train, cv=5, scoring='r2', n_jobs=-1)
# print("Cross-validation score is {score:.3f},"
# " standard deviation is {err:.3f}"
# .format(score = cv_score.mean(), err = cv_score.std()))
model = HistGradientBoostingRegressor(max_iter=300, l2_regularization=params[i][0], learning_rate=params[i][1], max_depth=int(params[i][2]))
model = model.fit(X_train, y_train)
y_pred_label = model.predict(X_test)
y_pred[label] = y_pred_label
print(label, ': finished! \n')
df_pred = pd.DataFrame(y_pred)
compression_options = dict(method='zip', archive_name='prediction.csv')
df_pred.to_csv('prediction.zip', index=False, float_format='%.3f', compression=compression_options)
print('All finished!')
(1, (0.05, 0.95), "'grid_resolution' must be strictly greater than 1")]
)
def test_grid_from_X_error(grid_resolution, percentiles, err_msg):
X = np.asarray([[1, 2], [3, 4]])
with pytest.raises(ValueError, match=err_msg):
_grid_from_X(
X, grid_resolution=grid_resolution, percentiles=percentiles
)
@pytest.mark.parametrize('target_feature', range(5))
@pytest.mark.parametrize('est, method', [
(LinearRegression(), 'brute'),
(GradientBoostingRegressor(random_state=0), 'brute'),
(GradientBoostingRegressor(random_state=0), 'recursion'),
(HistGradientBoostingRegressor(random_state=0), 'brute'),
(HistGradientBoostingRegressor(random_state=0), 'recursion')]
)
def test_partial_dependence_helpers(est, method, target_feature):
# Check that what is returned by _partial_dependence_brute or
# _partial_dependence_recursion is equivalent to manually setting a target
# feature to a given value, and computing the average prediction over all
# samples.
# This also checks that the brute and recursion methods give the same
# output.
# Note that even on the trainset, the brute and the recursion methods
# aren't always strictly equivalent, in particular when the slow method
# generates unrealistic samples that have low mass in the joint
# distribution of the input features, and when some of the features are
# dependent. Hence the high tolerance on the checks.
##################################### # Available optimisations on this machine. from mlprodict.testing.experimental_c_impl.experimental_c import code_optimisation print(code_optimisation()) ######################################## # Training and converting a model # +++++++++++++++++++++++++++++++ data = make_regression(50000, 20) X, y = data X_train, X_test, y_train, y_test = train_test_split(X, y) hgb = HistGradientBoostingRegressor(max_iter=100, max_depth=6) hgb.fit(X_train, y_train) print(hgb) ######################################## # Let's get more statistics about the model itself. pprint(analyze_model(hgb)) ################################# # And let's convert it. register_rewritten_operators() onx = to_onnx(hgb, X_train[:1].astype(numpy.float32)) oinf = OnnxInference(onx, runtime='python_compiled') print(oinf)
from sklearn.inspection import plot_partial_dependence from sklearn.ensemble import HistGradientBoostingRegressor from sklearn.datasets._california_housing import fetch_california_housing df = fetch_california_housing(as_frame=True) print(df) x = df['data'] y = df['target'] est = HistGradientBoostingRegressor().fit(x, y) est.score(x, y) features = ['HouseAge', 'HouseAge', ['MedInc', 'HouseAge']] plot_partial_dependence(est, x, features=features) """We can clearly see an interaction between the two features: for an Median income > 4.5, the House price is Dependent on HouseAge, for MedIncome < 4.5 NO STRONG depedence HousePricing and HouseAge. Makes sense ! Many Rich people Create more brand new Houses so they affect the House prices. if we lived in Sao Paolo the very low income will not affect the Relationship(Prices, Population) because everyone is poor. """ from matplotlib import pyplot as plt plt.gca() plt.show() """Disadvantages of PDP: - The realistic maximum number of features in a partial dependence function is two. - The assumption of independence is the biggest issue with PD plots. It is assumed that the feature(s) for which the partial dependence is computed are not correlated with other features. One solution to this problem is Accumulated Local Effect plots or short ALE plots that work with the conditional instead of the marginal distribution. -By plotting the individual conditional expectation curves instead of the aggregated line,
from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingRegressor
hist = HistGradientBoostingRegressor(random_state=42)
hist.fit(X_train, y_train)
hist_pred = hist.predict(X_test)
compute_metrics(y_test, hist_pred)
hist_poisson = HistGradientBoostingRegressor(loss='poisson', random_state=42)
hist_poisson.fit(X_train, y_train)
hist_poisson_pred = hist_poisson.predict(X_test)
compute_metrics(y_test, hist_poisson_pred)
fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(18, 6), sharey=True)
ax1.hist(y_test, bins=30, alpha=0.5)
ax1.set_title("Test data")
ax2.hist(hist_pred, bins=30, alpha=0.5)
ax2.set_title("Default Hist")
ax3.hist(hist_poisson_pred, bins=30, alpha=0.5)
ax3.set_title("Poisson Hist");
def pick_algorithm():
# Choose the right model
pipelines = []
pipelines.append(('ScaledLR',
Pipeline([('Scaler', StandardScaler()),
('LR', LinearRegression())])))
pipelines.append(('ScaledSGDR',
Pipeline([('Scaler', StandardScaler()),
('SGDR', SGDRegressor())])))
pipelines.append(('ScaledHR',
Pipeline([('Scaler', StandardScaler()),
('HR', HuberRegressor())])))
pipelines.append(('ScaledLARS',
Pipeline([('Scaler', StandardScaler()),
('LARS', Lars())])))
pipelines.append(('ScaledLL',
Pipeline([('Scaler', StandardScaler()),
('LL', LassoLars())])))
pipelines.append(('ScaledORP',
Pipeline([('Scaler', StandardScaler()),
('ORP', OrthogonalMatchingPursuit())])))
pipelines.append(('ScaledPAR',
Pipeline([('Scaler', StandardScaler()),
('PAR', PassiveAggressiveRegressor())])))
pipelines.append(('ScaledBR',
Pipeline([('Scaler', StandardScaler()),
('BR', BayesianRidge())])))
pipelines.append(('ScaledRIDGE',
Pipeline([('Scaler', StandardScaler()),
('RIDGE', Ridge())])))
pipelines.append(('ScaledLASSO',
Pipeline([('Scaler', StandardScaler()),
('LASSO', Lasso())])))
pipelines.append(('ScaledEN',
Pipeline([('Scaler', StandardScaler()),
('EN', ElasticNet())])))
pipelines.append(('ScaledKNN',
Pipeline([('Scaler', StandardScaler()),
('KNN', KNeighborsRegressor())])))
pipelines.append(('ScaledCART',
Pipeline([('Scaler', StandardScaler()),
('CART', DecisionTreeRegressor())])))
pipelines.append(('ScaledGBM',
Pipeline([('Scaler', StandardScaler()),
('GBM', GradientBoostingRegressor())])))
pipelines.append(('ScaledABR',
Pipeline([('Scaler', StandardScaler()),
('ABR', AdaBoostRegressor())])))
pipelines.append(('ScaledBAR',
Pipeline([('Scaler', StandardScaler()),
('BAR', BaggingRegressor())])))
pipelines.append(('ScaledHGBR',
Pipeline([('Scaler', StandardScaler()),
('HGBR', HistGradientBoostingRegressor())])))
results = []
names = []
for name, model in pipelines:
kfold = KFold(n_splits=10, random_state=args.seed)
cv_results = cross_val_score(model,
xtrain,
ytrain,
cv=kfold,
scoring='neg_mean_squared_error')
results.append(cv_results)
names.append(name)
msg = f"{name}: {cv_results.mean()}"
print(msg)
# ESTIMATORS #
# =========================================================================== #
regressors = {}
regressors.update({"Linear Regression": LinearRegression()})
regressors.update({"Lasso": Lasso()})
regressors.update({"Ridge": Ridge()})
regressors.update({"ElasticNet": ElasticNet()})
ensembles = {}
ensembles.update({"AdaBoost": AdaBoostRegressor()})
ensembles.update({"Bagging": BaggingRegressor()})
ensembles.update({"Extra Trees": ExtraTreesRegressor()})
ensembles.update({"Gradient Boosting": GradientBoostingRegressor()})
ensembles.update({"Random Forest": RandomForestRegressor()})
ensembles.update(
{"Histogram Gradient Boosting": HistGradientBoostingRegressor()})
# =========================================================================== #
# HYPERPARAMETERS #
# =========================================================================== #
# Parameter Grid
regressor_parameters = {}
regressor_parameters.update(
{"Linear Regression": {
"estimator__normalize": [False]
}})
regressor_parameters.update({
"Lasso": {
"estimator__alpha":
[1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.25, 0.50, 0.75, 1.0],
"estimator__n_jobs": [-1]