def fit(self, X, Y):
if self.loss == "mse":
loss = MSELoss()
elif self.loss == "crossentropy":
loss = CrossEntropyLoss()
# convert Y to one-hot encoding
if self.classifier:
Y = to_one_hot(Y.flatten())
else:
# if the shape of Y is like (N,), then we need to convert it to be (N,1)
Y = Y.reshape(-1, 1) if len(Y.shape) == 1 else Y
N, M = X.shape
# out_dims is the number of classes in the outcome. 1 for continuous Y
self.out_dims = Y.shape[1]
# Record all the learners (all the trees)
self.learners = np.empty((self.n_iter, self.out_dims), dtype=object)
# weights for each prediction (since we don't do the linear search step, we just put the learning_rate here)
# The very first iteration has weights equals to 1 (so we don't multiply self.learning_rate)
self.weights = np.ones((self.n_iter, self.out_dims))
self.weights[1:, :] *= self.learning_rate
# Prediction values, N samples, and each samples has self.out_dims dimensions
Y_pred = np.zeros((N, self.out_dims))
# Very first iteration, use mean to predict
for k in range(self.out_dims):
t = loss.base_estimator()
t.fit(X, Y[:, k])
Y_pred[:, k] = t.predict(X)
self.learners[0, k] = t
# Incrementally fit each learner on the negative gradient of the loss
# wrt the previous fit (pseudo-residuals)
for i in range(1, self.n_iter):
for k in range(self.out_dims):
y, y_pred = Y[:, k], Y_pred[:, k]
neg_grad = -1 * loss.grad(y, y_pred)
# use MSE as the surrogate loss when fitting to negative gradients
t = DecisionTree(classifier=False,
max_depth=self.max_depth,
criterion="mse")
# fit X to negative gradients of the current loss function
t.fit(X, neg_grad)
self.learners[i, k] = t
# compute step size and weight for the current learner
step = 1.0
h_pred = t.predict(X)
# We ignore the linear search step
# update weights and our overall prediction for Y
self.weights[i, k] *= step
Y_pred[:, k] += self.weights[i, k] * h_pred
def fit(self, X, Y):
"""
Fit the gradient boosted decision trees on a dataset
:param X:
:param Y:
:return:
"""
if self.loss == "mse":
loss = MSELoss()
elif self.loss == "crossentropy":
loss = CrossEntropyLoss()
if self.classifier:
Y = to_one_hot(Y.flatten())
else:
Y = Y.reshape(-1, 1) if len(Y.shape) == 1 else Y
N, M = X.shape
self.out_dims = Y.shape[1]
self.learners = np.empty((self.n_iter, self.out_dims), dtype=object)
self.weights = np.ones((self.n_iter, self.out_dims))
self.weights[1:, :] = self.learning_rate
# fit the base estimator
Y_pred = np.zeros((N, self.out_dims))
for k in range(self.out_dims):
t = loss.base_estimator()
t.fit(X, Y[:, k])
Y_pred[:, k] += t.predict(X)
self.learners[0, k] = t
# incrementally fit each learner on the negative gradient of the loss
for i in range(1, self.n_iter):
for k in range(self.out_dims):
y, y_pred = Y[:, k], Y_pred[:, k]
neg_grad = -1 * loss.grad(y, y_pred)
t = DecisionTree(classifier=False,
max_depth=self.max_depth,
criterion="mse")
t.fit(X, neg_grad)
self.learners[i, k] = t
step = 1.0
h_pred = t.predict(X)
if self.step_size == "adaptive":
step = loss.line_search(y, y_pred, h_pred)
self.weights[i, k] *= step
Y_pred[:, k] += self.weights[i, k] * h_pred
def fit(self, X, Y):
if self.loss == "mse":
loss = MSELoss()
elif self.loss == "crossentropy":
loss = CrossEntropyLoss()
# convert Y to one_hot if not already
if self.classifier:
Y = to_one_hot(Y.flatten())
else:
Y = Y.reshape(-1, 1) if len(Y.shape) == 1 else Y
N, M = X.shape
self.out_dims = Y.shape[1]
self.learners = np.empty((self.n_iter, self.out_dims), dtype=object)
self.weights = np.ones((self.n_iter, self.out_dims))
self.weights[1:, :] *= self.learning_rate
# fit the base estimator
Y_pred = np.zeros((N, self.out_dims))
for k in range(self.out_dims):
t = loss.base_estimator()
t.fit(X, Y[:, k])
Y_pred[:, k] += t.predict(X)
self.learners[0, k] = t
# incrementally fit each learner on the negative gradient of the loss
# wrt the previous fit (pseudo-residuals)
for i in range(1, self.n_iter):
for k in range(self.out_dims):
y, y_pred = Y[:, k], Y_pred[:, k]
neg_grad = -1 * loss.grad(y, y_pred)
# use MSE as the surrogate loss when fitting to negative gradients
t = DecisionTree(classifier=False,
max_depth=self.max_depth,
criterion="mse")
# fit current learner to negative gradients
t.fit(X, neg_grad)
self.learners[i, k] = t
# compute step size and weight for the current learner
step = 1.0
h_pred = t.predict(X)
if self.step_size == "adaptive":
step = loss.line_search(y, y_pred, h_pred)
# update weights and our overall prediction for Y
self.weights[i, k] *= step
Y_pred[:, k] += self.weights[i, k] * h_pred