class AbstractGradientBoostingClassifier(BaseEstimator, ClassifierMixin):
def __init__(self, loss=None,
n_estimators=100,
learning_rate=0.1,
subsample=1.0,
train_variables=None,
random_state=None,
n_threads=1,
dtype=DTYPE):
"""This version of gradient boosting supports only two-class classification and only special losses
derived from AbstractLossFunction.
There are some methods that should be overriden in descendants.
:type loss: AbstractLossFunction, by default AdaLossFunction is used
"""
self.loss = loss
self.n_estimators = n_estimators
self.learning_rate = learning_rate
self.subsample = subsample
self.train_variables = train_variables
self.random_state = random_state
self.initial_prediction = 0.
self.dtype = dtype
self.n_threads = n_threads
def _check_params(self):
if self.loss is None:
self.loss = AdaLossFunction()
# Losses from sklearn are not allowed
assert isinstance(self.loss, AbstractLossFunction), \
'LossFunction should be derived from AbstractLossFunction'
assert self.n_estimators > 0, 'n_estimators should be positive'
self.random_state = check_random_state(self.random_state)
assert 0 < self.subsample <= 1.0, 'subsample should be in the interval (0, 1]'
def _create_estimator(self, stage):
raise NotImplementedError('Should be overriden in descendants')
def _fit_estimator(self, estimator, X, y, sample_weight, residual, mask):
""" mask - which events to use in training """
# TODO do we need check_input=false for trees?
estimator.fit(X[mask, :], residual[mask], sample_weight=sample_weight[mask])
def _update_estimator(self, estimator, X, y, sample_weight, residual, y_pred, mask):
pass
def _prepare_data_for_fitting(self, X, y, sample_weight):
"""By default the same format used as for trees """
X = self.get_train_vars(X)
X, y = check_arrays(X, y, dtype=self.dtype, sparse_format="dense", check_ccontiguous=True)
return X, y, sample_weight
@staticmethod
def _initial_data_check(X, y, sample_weight):
sample_weight = check_sample_weight(y, sample_weight=sample_weight)
assert len(X) == len(y), 'Different lengths of X and y'
X = pandas.DataFrame(X)
y = numpy.array(column_or_1d(y), dtype=int)
assert numpy.all(numpy.in1d(y, [0, 1])), 'Only two-class classification supported'
return X, y, sample_weight
def _prepare_initial_predictions(self, X, y, sample_weight):
self.initial_prediction = logit(numpy.average(y, weights=sample_weight))
def _compute_initial_predictions(self, X):
return numpy.zeros(len(X), dtype='float') + self.initial_prediction
def _generate_mask(self, length, subsample):
if subsample == 1.0:
return slice(None, None, None)
else:
n_sampled_events = int(subsample * length)
return self.random_state.choice(length, n_sampled_events, replace=True)
def fit(self, X, y, sample_weight=None):
X, y, sample_weight = self._initial_data_check(X, y, sample_weight)
self._check_params()
loss_weight = numpy.ones(len(sample_weight))
tree_weight = sample_weight
if False:
loss_weight, tree_weight = tree_weight, loss_weight
self.loss = copy.copy(self.loss)
self.loss.fit(X, y, sample_weight=loss_weight)
X, y, sample_weight = self._prepare_data_for_fitting(X, y, sample_weight)
self._prepare_initial_predictions(X, y, sample_weight)
y_pred = self._compute_initial_predictions(X)
self.estimators = []
self.scores = []
# pool = ThreadPool(processes=self.n_threads)
lock = Lock()
train_params = [self, X, y, tree_weight, y_pred, lock]
# TODO use threading
# pool.map(_train_one_classifier, [train_params] * self.n_estimators, chunksize=1)
map(_train_one_classifier, [train_params] * self.n_estimators)
return self
def get_train_vars(self, X):
if self.train_variables is None:
return numpy.array(X)
else:
return numpy.array(X.loc[:, self.train_variables])
@staticmethod
def score_to_proba(score):
result = numpy.zeros([len(score), 2], dtype=float)
result[:, 1] = sigmoid_function(score, width=1.)
result[:, 0] = 1. - result[:, 1]
return result
def staged_predict_score(self, X):
X = self.get_train_vars(X)
y_pred = self._compute_initial_predictions(X)
for estimator in self.estimators:
y_pred += self.learning_rate * estimator.predict(X)
yield y_pred
def predict_score(self, X):
result = None
for score in self.staged_predict_score(X):
result = score
return result
def staged_predict_proba(self, X):
for score in self.staged_predict_score(X):
yield self.score_to_proba(score)
def predict_proba(self, X):
return self.score_to_proba(self.predict_score(X))
def predict(self, X):
return numpy.argmax(self.predict_proba(X), axis=1)
class AbstractGradientBoostingClassifier(BaseEstimator, ClassifierMixin):
def __init__(self,
loss=None,
n_estimators=100,
learning_rate=0.1,
subsample=1.0,
train_variables=None,
random_state=None,
n_threads=1,
dtype=DTYPE):
"""This version of gradient boosting supports only two-class classification and only special losses
derived from AbstractLossFunction.
There are some methods that should be overriden in descendants.
:type loss: AbstractLossFunction, by default AdaLossFunction is used
"""
self.loss = loss
self.n_estimators = n_estimators
self.learning_rate = learning_rate
self.subsample = subsample
self.train_variables = train_variables
self.random_state = random_state
self.initial_prediction = 0.
self.dtype = dtype
self.n_threads = n_threads
def _check_params(self):
if self.loss is None:
self.loss = AdaLossFunction()
# Losses from sklearn are not allowed
assert isinstance(self.loss, AbstractLossFunction), \
'LossFunction should be derived from AbstractLossFunction'
assert self.n_estimators > 0, 'n_estimators should be positive'
self.random_state = check_random_state(self.random_state)
assert 0 < self.subsample <= 1.0, 'subsample should be in the interval (0, 1]'
def _create_estimator(self, stage):
raise NotImplementedError('Should be overriden in descendants')
def _fit_estimator(self, estimator, X, y, sample_weight, residual, mask):
""" mask - which events to use in training """
# TODO do we need check_input=false for trees?
estimator.fit(X[mask, :],
residual[mask],
sample_weight=sample_weight[mask])
def _update_estimator(self, estimator, X, y, sample_weight, residual,
y_pred, mask):
pass
def _prepare_data_for_fitting(self, X, y, sample_weight):
"""By default the same format used as for trees """
X = self.get_train_vars(X)
X, y = check_arrays(X, y)
X = X.astype(self.dtype)
return X, y, sample_weight
@staticmethod
def _initial_data_check(X, y, sample_weight):
sample_weight = check_sample_weight(y, sample_weight=sample_weight)
assert len(X) == len(y), 'Different lengths of X and y'
X = pandas.DataFrame(X)
y = numpy.array(column_or_1d(y), dtype=int)
assert numpy.all(numpy.in1d(
y, [0, 1])), 'Only two-class classification supported'
return X, y, sample_weight
def _prepare_initial_predictions(self, X, y, sample_weight):
self.initial_prediction = logit(numpy.average(y,
weights=sample_weight))
def _compute_initial_predictions(self, X):
return numpy.zeros(len(X), dtype='float') + self.initial_prediction
def _generate_mask(self, length, subsample):
if subsample == 1.0:
return slice(None, None, None)
else:
n_sampled_events = int(subsample * length)
return self.random_state.choice(length,
n_sampled_events,
replace=True)
def fit(self, X, y, sample_weight=None):
X, y, sample_weight = self._initial_data_check(X, y, sample_weight)
self._check_params()
loss_weight = numpy.ones(len(sample_weight))
tree_weight = sample_weight
if False:
loss_weight, tree_weight = tree_weight, loss_weight
self.loss = copy.copy(self.loss)
self.loss.fit(X, y, sample_weight=loss_weight)
X, y, sample_weight = self._prepare_data_for_fitting(
X, y, sample_weight)
self._prepare_initial_predictions(X, y, sample_weight)
y_pred = self._compute_initial_predictions(X)
self.estimators = []
self.scores = []
# pool = ThreadPool(processes=self.n_threads)
lock = Lock()
train_params = [self, X, y, tree_weight, y_pred, lock]
# TODO use threading
# pool.map(_train_one_classifier, [train_params] * self.n_estimators, chunksize=1)
map(_train_one_classifier, [train_params] * self.n_estimators)
return self
def get_train_vars(self, X):
if self.train_variables is None:
return numpy.array(X)
else:
return numpy.array(X.loc[:, self.train_variables])
@staticmethod
def score_to_proba(score):
result = numpy.zeros([len(score), 2], dtype=float)
result[:, 1] = sigmoid_function(score, width=1.)
result[:, 0] = 1. - result[:, 1]
return result
def staged_predict_score(self, X):
X = self.get_train_vars(X)
y_pred = self._compute_initial_predictions(X)
for estimator in self.estimators:
y_pred += self.learning_rate * estimator.predict(X)
yield y_pred
def predict_score(self, X):
result = None
for score in self.staged_predict_score(X):
result = score
return result
def staged_predict_proba(self, X):
for score in self.staged_predict_score(X):
yield self.score_to_proba(score)
def predict_proba(self, X):
return self.score_to_proba(self.predict_score(X))
def predict(self, X):
return numpy.argmax(self.predict_proba(X), axis=1)
