def __init__(self, reg_factor, n_iterations, learning_rate, gradient_descent):
self.w = None
self.n_iterations = n_iterations
self.learning_rate = learning_rate
self.gradient_descent = gradient_descent
self.reg_factor = reg_factor
self.square_loss = SquareLoss()
def __init__(self, n_estimators, learning_rate, min_samples_split,
min_impurity, max_depth, regression, debug):
self.n_estimators = n_estimators
self.learning_rate = learning_rate
self.min_samples_split = min_samples_split
self.min_impurity = min_impurity
self.max_depth = max_depth
self.init_estimate = None
self.regression = regression
self.debug = debug
self.multipliers = []
self.bar = progressbar.ProgressBar(widgets=bar_widgets)
# Square loss for regression
# Log loss for classification
self.loss = SquareLoss()
if not self.regression:
self.loss = CrossEntropy()
# Initialize regression trees
self.trees = []
for _ in range(n_estimators):
tree = RegressionTree(min_samples_split=self.min_samples_split,
min_impurity=min_impurity,
max_depth=self.max_depth)
self.trees.append(tree)
def __init__(self, n_estimators, learning_rate, min_samples_split,
min_impurity, max_depth, regression, debug):
self.n_estimators = n_estimators
self.learning_rate = learning_rate
self.min_samples_split = min_samples_split
self.min_impurity = min_impurity
self.max_depth = max_depth
self.init_estimate = None
self.regression = regression
self.debug = debug
self.multipliers = []
# Square loss for regression
# Log loss for classification
self.loss = SquareLoss(grad_wrt_theta=False)
if not self.regression:
self.loss = LogisticLoss(grad_wrt_theta=False)
# Initialize regression trees
self.trees = []
for _ in range(n_estimators):
tree = RegressionTree(
min_samples_split=self.min_samples_split,
min_impurity=min_impurity,
max_depth=self.max_depth)
self.trees.append(tree)
class Regression(object):
""" Base regression model. Models the relationship between a scalar dependent variable y and the independent
variables X.
Parameters:
-----------
reg_factor: float
The factor that will determine the amount of regularization and feature
shrinkage.
n_iterations: float
The number of training iterations the algorithm will tune the weights for.
learning_rate: float
The step length that will be used when updating the weights.
gradient_descent: boolean
True or false depending if gradient descent should be used when training. If
false then we use batch optimization by least squares.
"""
def __init__(self, reg_factor, n_iterations, learning_rate,
gradient_descent):
self.w = None
self.n_iterations = n_iterations
self.learning_rate = learning_rate
self.gradient_descent = gradient_descent
self.reg_factor = reg_factor
self.square_loss = SquareLoss()
def fit(self, X, y):
# Insert constant ones as first column (for bias weights)
X = np.insert(X, 0, 1, axis=1)
n_features = np.shape(X)[1]
# Get weights by gradient descent opt.
if self.gradient_descent:
# Initial weights randomly [0, 1]
self.w = np.random.random((n_features, ))
# Do gradient descent for n_iterations
for _ in range(self.n_iterations):
grad_w = self.square_loss.gradient(
y, X, self.w) + self.reg_factor * self.w
self.w -= self.learning_rate * grad_w
# Get weights by least squares (by pseudoinverse)
else:
U, S, V = np.linalg.svd(
X.T.dot(X) + self.reg_factor * np.identity(n_features))
S = np.diag(S)
X_sq_reg_inv = V.dot(np.linalg.pinv(S)).dot(U.T)
self.w = X_sq_reg_inv.dot(X.T).dot(y)
def predict(self, X):
# Insert constant ones for bias weights
X = np.insert(X, 0, 1, axis=1)
y_pred = X.dot(self.w)
return y_pred
class Regression(object):
""" Base regression model. Models the relationship between a scalar dependent variable y and the independent
variables X.
Parameters:
-----------
reg_factor: float
The factor that will determine the amount of regularization and feature
shrinkage.
n_iterations: float
The number of training iterations the algorithm will tune the weights for.
learning_rate: float
The step length that will be used when updating the weights.
gradient_descent: boolean
True or false depending if gradient descent should be used when training. If
false then we use batch optimization by least squares.
"""
def __init__(self, reg_factor, n_iterations, learning_rate, gradient_descent):
self.w = None
self.n_iterations = n_iterations
self.learning_rate = learning_rate
self.gradient_descent = gradient_descent
self.reg_factor = reg_factor
self.square_loss = SquareLoss()
def fit(self, X, y):
# Insert constant ones as first column (for bias weights)
X = np.insert(X, 0, 1, axis=1)
n_features = np.shape(X)[1]
# Get weights by gradient descent opt.
if self.gradient_descent:
# Initial weights randomly [0, 1]
self.w = np.random.random((n_features, ))
# Do gradient descent for n_iterations
for _ in range(self.n_iterations):
grad_w = self.square_loss.gradient(y, X, self.w) + self.reg_factor * self.w
self.w -= self.learning_rate * grad_w
# Get weights by least squares (by pseudoinverse)
else:
U, S, V = np.linalg.svd(
X.T.dot(X) + self.reg_factor * np.identity(n_features))
S = np.diag(S)
X_sq_reg_inv = V.dot(np.linalg.pinv(S)).dot(U.T)
self.w = X_sq_reg_inv.dot(X.T).dot(y)
def predict(self, X):
# Insert constant ones for bias weights
X = np.insert(X, 0, 1, axis=1)
y_pred = X.dot(self.w)
return y_pred