class PrudentialRegressorCVO(BaseEstimator, RegressorMixin):
def __init__(self,
objective='reg:linear',
learning_rate=0.045,
min_child_weight=50,
subsample=0.8,
colsample_bytree=0.7,
max_depth=7,
n_estimators=700,
nthread=-1,
seed=0,
n_buckets=8,
initial_params=[-1.5, -2.6, -3.6, -1.2, -0.8, 0.04, 0.7, 3.6,
#1., 2., 3., 4., 5., 6., 7.
],
minimizer='BFGS',
scoring=NegQWKappaScorer):
self.objective = objective
self.learning_rate = learning_rate
self.min_child_weight = min_child_weight
self.subsample = subsample
self.colsample_bytree = colsample_bytree
self.max_depth = max_depth
self.n_estimators = n_estimators
self.nthread = nthread
self.seed = seed
self.n_buckets = n_buckets
self.initial_params = initial_params
self.minimizer = minimizer
self.scoring = scoring
return
def fit(self, X, y):
from xgboost import XGBRegressor
if not KAGGLE:
from OptimizedOffsetRegressor import DigitizedOptimizedOffsetRegressor
#from OptimizedOffsetRegressor import FullDigitizedOptimizedOffsetRegressor
#self.off = FullDigitizedOptimizedOffsetRegressor(n_buckets=self.n_buckets,
# basinhopping=True,
"""
2 / 5
grid scores:
mean: 0.65531, std: 0.00333, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65531
3 / 5
grid scores:
mean: 0.65474, std: 0.00308, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65474
4 / 5
grid scores:
mean: 0.65490, std: 0.00302, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65490
2 / 10
grid scores:
mean: 0.65688, std: 0.00725, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65688
3 / 10
grid scores:
mean: 0.65705, std: 0.00714, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65705
4 / 10
grid scores:
mean: 0.65643, std: 0.00715, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65643
5 / 10
grid scores:
mean: 0.65630, std: 0.00699, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65630
"""
from sklearn.cross_validation import StratifiedKFold
kf = StratifiedKFold(y, n_folds=2)
print(kf)
params = []
for itrain, itest in kf:
ytrain = y[itrain]
Xtrain = X.iloc[list(itrain)]
ytest = y[itest]
Xtest = X.iloc[list(itest)]
self.xgb = XGBRegressor(
objective=self.objective,
learning_rate=self.learning_rate,
min_child_weight=self.min_child_weight,
subsample=self.subsample,
colsample_bytree=self.colsample_bytree,
max_depth=self.max_depth,
n_estimators=self.n_estimators,
nthread=self.nthread,
missing=0.0,
seed=self.seed)
self.xgb.fit(Xtrain, ytrain)
te_y_hat = self.xgb.predict(Xtest,
ntree_limit=self.xgb.booster().best_iteration)
print('XGB Test score is:', -self.scoring(te_y_hat, ytest))
self.off = DigitizedOptimizedOffsetRegressor(n_buckets=self.n_buckets,
initial_params=self.initial_params,
minimizer=self.minimizer,
scoring=self.scoring)
self.off.fit(te_y_hat, ytest)
print("Offsets:", self.off.params)
params += [list(self.off.params)]
pass
from numpy import array
self.off.params = array(params).mean(axis=0)
print("Mean Offsets:", self.off.params)
self.xgb.fit(X, y)
return self
def predict(self, X):
from numpy import clip
te_y_hat = self.xgb.predict(X, ntree_limit=self.xgb.booster().best_iteration)
return clip(self.off.predict(te_y_hat), 1, 8)
pass
class PrudentialRegressorCVO(BaseEstimator, RegressorMixin):
def __init__(
self,
objective='reg:linear',
learning_rate=0.045,
min_child_weight=50,
subsample=0.8,
colsample_bytree=0.7,
max_depth=7,
n_estimators=700,
nthread=-1,
seed=0,
n_buckets=8,
initial_params=[
-1.5,
-2.6,
-3.6,
-1.2,
-0.8,
0.04,
0.7,
3.6,
#1., 2., 3., 4., 5., 6., 7.
],
minimizer='BFGS',
scoring=NegQWKappaScorer):
self.objective = objective
self.learning_rate = learning_rate
self.min_child_weight = min_child_weight
self.subsample = subsample
self.colsample_bytree = colsample_bytree
self.max_depth = max_depth
self.n_estimators = n_estimators
self.nthread = nthread
self.seed = seed
self.n_buckets = n_buckets
self.initial_params = initial_params
self.minimizer = minimizer
self.scoring = scoring
return
def fit(self, X, y):
from xgboost import XGBRegressor
if not KAGGLE:
from OptimizedOffsetRegressor import DigitizedOptimizedOffsetRegressor
#from OptimizedOffsetRegressor import FullDigitizedOptimizedOffsetRegressor
#self.off = FullDigitizedOptimizedOffsetRegressor(n_buckets=self.n_buckets,
# basinhopping=True,
"""
2 / 5
grid scores:
mean: 0.65531, std: 0.00333, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65531
3 / 5
grid scores:
mean: 0.65474, std: 0.00308, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65474
4 / 5
grid scores:
mean: 0.65490, std: 0.00302, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65490
2 / 10
grid scores:
mean: 0.65688, std: 0.00725, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65688
3 / 10
grid scores:
mean: 0.65705, std: 0.00714, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65705
4 / 10
grid scores:
mean: 0.65643, std: 0.00715, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65643
5 / 10
grid scores:
mean: 0.65630, std: 0.00699, params: {'n_estimators': 700, 'subsample': 0.9, 'colsample_bytree': 0.67, 'max_depth': 6, 'min_child_weight': 240}
best score: 0.65630
"""
from sklearn.cross_validation import StratifiedKFold
kf = StratifiedKFold(y, n_folds=2)
print(kf)
params = []
for itrain, itest in kf:
ytrain = y[itrain]
Xtrain = X.iloc[list(itrain)]
ytest = y[itest]
Xtest = X.iloc[list(itest)]
self.xgb = XGBRegressor(objective=self.objective,
learning_rate=self.learning_rate,
min_child_weight=self.min_child_weight,
subsample=self.subsample,
colsample_bytree=self.colsample_bytree,
max_depth=self.max_depth,
n_estimators=self.n_estimators,
nthread=self.nthread,
missing=0.0,
seed=self.seed)
self.xgb.fit(Xtrain, ytrain)
te_y_hat = self.xgb.predict(
Xtest, ntree_limit=self.xgb.booster().best_iteration)
print('XGB Test score is:', -self.scoring(te_y_hat, ytest))
self.off = DigitizedOptimizedOffsetRegressor(
n_buckets=self.n_buckets,
initial_params=self.initial_params,
minimizer=self.minimizer,
scoring=self.scoring)
self.off.fit(te_y_hat, ytest)
print("Offsets:", self.off.params)
params += [list(self.off.params)]
pass
from numpy import array
self.off.params = array(params).mean(axis=0)
print("Mean Offsets:", self.off.params)
self.xgb.fit(X, y)
return self
def predict(self, X):
from numpy import clip
te_y_hat = self.xgb.predict(
X, ntree_limit=self.xgb.booster().best_iteration)
return clip(self.off.predict(te_y_hat), 1, 8)
pass
class PrudentialRegressor(BaseEstimator, RegressorMixin):
def __init__(self,
objective='reg:linear',
learning_rate=0.045,
min_child_weight=50,
subsample=0.8,
colsample_bytree=0.7,
max_depth=7,
n_estimators=700,
nthread=-1,
seed=0,
n_buckets=8,
initial_params=[-1.5, -2.6, -3.6, -1.2, -0.8, 0.04, 0.7, 3.6,
#1., 2., 3., 4., 5., 6., 7.
],
minimizer='BFGS',
scoring=NegQWKappaScorer):
self.objective = objective
self.learning_rate = learning_rate
self.min_child_weight = min_child_weight
self.subsample = subsample
self.colsample_bytree = colsample_bytree
self.max_depth = max_depth
self.n_estimators = n_estimators
self.nthread = nthread
self.seed = seed
self.n_buckets = n_buckets
self.initial_params = initial_params
self.minimizer = minimizer
self.scoring = scoring
return
def fit(self, X, y):
from xgboost import XGBRegressor
if not KAGGLE:
from OptimizedOffsetRegressor import DigitizedOptimizedOffsetRegressor
self.xgb = XGBRegressor(
objective=self.objective,
learning_rate=self.learning_rate,
min_child_weight=self.min_child_weight,
subsample=self.subsample,
colsample_bytree=self.colsample_bytree,
max_depth=self.max_depth,
n_estimators=self.n_estimators,
nthread=self.nthread,
missing=0.0,
seed=self.seed)
#from OptimizedOffsetRegressor import FullDigitizedOptimizedOffsetRegressor
#self.off = FullDigitizedOptimizedOffsetRegressor(n_buckets=self.n_buckets,
# basinhopping=True,
self.off = DigitizedOptimizedOffsetRegressor(n_buckets=self.n_buckets,
initial_params=self.initial_params,
minimizer=self.minimizer,
scoring=self.scoring)
self.xgb.fit(X, y)
tr_y_hat = self.xgb.predict(X,
ntree_limit=self.xgb.booster().best_iteration)
print('Train score is:', -self.scoring(tr_y_hat, y))
self.off.fit(tr_y_hat, y)
print("Offsets:", self.off.params)
return self
def predict(self, X):
from numpy import clip
te_y_hat = self.xgb.predict(X, ntree_limit=self.xgb.booster().best_iteration)
return clip(self.off.predict(te_y_hat), 1, 8)
pass
class PrudentialRegressor(BaseEstimator, RegressorMixin):
def __init__(
self,
objective='reg:linear',
learning_rate=0.045,
min_child_weight=50,
subsample=0.8,
colsample_bytree=0.7,
max_depth=7,
n_estimators=700,
nthread=-1,
seed=0,
n_buckets=8,
initial_params=[
-1.5,
-2.6,
-3.6,
-1.2,
-0.8,
0.04,
0.7,
3.6,
#1., 2., 3., 4., 5., 6., 7.
],
minimizer='BFGS',
scoring=NegQWKappaScorer):
self.objective = objective
self.learning_rate = learning_rate
self.min_child_weight = min_child_weight
self.subsample = subsample
self.colsample_bytree = colsample_bytree
self.max_depth = max_depth
self.n_estimators = n_estimators
self.nthread = nthread
self.seed = seed
self.n_buckets = n_buckets
self.initial_params = initial_params
self.minimizer = minimizer
self.scoring = scoring
return
def fit(self, X, y):
from xgboost import XGBRegressor
if not KAGGLE:
from OptimizedOffsetRegressor import DigitizedOptimizedOffsetRegressor
self.xgb = XGBRegressor(objective=self.objective,
learning_rate=self.learning_rate,
min_child_weight=self.min_child_weight,
subsample=self.subsample,
colsample_bytree=self.colsample_bytree,
max_depth=self.max_depth,
n_estimators=self.n_estimators,
nthread=self.nthread,
missing=0.0,
seed=self.seed)
#from OptimizedOffsetRegressor import FullDigitizedOptimizedOffsetRegressor
#self.off = FullDigitizedOptimizedOffsetRegressor(n_buckets=self.n_buckets,
# basinhopping=True,
self.off = DigitizedOptimizedOffsetRegressor(
n_buckets=self.n_buckets,
initial_params=self.initial_params,
minimizer=self.minimizer,
scoring=self.scoring)
self.xgb.fit(X, y)
tr_y_hat = self.xgb.predict(
X, ntree_limit=self.xgb.booster().best_iteration)
print('Train score is:', -self.scoring(tr_y_hat, y))
self.off.fit(tr_y_hat, y)
print("Offsets:", self.off.params)
return self
def predict(self, X):
from numpy import clip
te_y_hat = self.xgb.predict(
X, ntree_limit=self.xgb.booster().best_iteration)
return clip(self.off.predict(te_y_hat), 1, 8)
pass