def fit(self,
X,
y,
n_iterations=3000,
learning_rate=0.01,
plot_errors=False):
# Convert the nominal y values to binary
y = categorical_to_binary(y)
n_samples, n_features = np.shape(X)
n_outputs = np.shape(y)[1]
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1 / math.sqrt(n_features)
b = -a
self.W = (b - a) * np.random.random((n_features, self.n_hidden)) + a
self.biasW = (b - a) * np.random.random((1, self.n_hidden)) + a
self.V = (b - a) * np.random.random((self.n_hidden, n_outputs)) + a
self.biasV = (b - a) * np.random.random((1, n_outputs)) + a
errors = []
for i in range(n_iterations):
# Calculate hidden layer
hidden_input = X.dot(self.W) + self.biasW
# Calculate output of hidden neurons
hidden_output = sigmoid(hidden_input)
# Calculate output layer
output_layer_input = hidden_output.dot(self.V) + self.biasV
output_layer_pred = sigmoid(output_layer_input)
# Calculate the error
error = y - output_layer_pred
mse = np.mean(np.power(error, 2))
errors.append(mse)
# Calculate loss gradients:
# Gradient of squared loss w.r.t each parameter
v_gradient = -2 * (y - output_layer_pred) * \
sigmoid_gradient(output_layer_input)
biasV_gradient = v_gradient
w_gradient = v_gradient.dot(
self.V.T) * sigmoid_gradient(hidden_input)
biasW_gradient = w_gradient
# Update weights
# Move against the gradient to minimize loss
self.V -= learning_rate * hidden_output.T.dot(v_gradient)
self.biasV -= learning_rate * np.ones(
(1, n_samples)).dot(biasV_gradient)
self.W -= learning_rate * X.T.dot(w_gradient)
self.biasW -= learning_rate * np.ones(
(1, n_samples)).dot(biasW_gradient)
# Plot the training error
if plot_errors:
plt.plot(range(n_iterations), errors)
plt.ylabel('Training Error')
plt.xlabel('Iterations')
plt.show()
def fit(self, X, y, n_iterations=3000, learning_rate=0.01, plot_errors=False):
X_train = np.array(X, dtype=float)
# Convert the nominal y values to binary
y_train = categorical_to_binary(y)
n_samples, n_features = np.shape(X_train)
n_outputs = np.shape(y_train)[1]
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1/math.sqrt(n_features)
b = -a
self.W = (b-a)*np.random.random((n_features, self.n_hidden)) + a
self.biasW = (b-a)*np.random.random((1, self.n_hidden)) + a
self.V = (b-a)*np.random.random((self.n_hidden, n_outputs)) + a
self.biasV = (b-a)*np.random.random((1, n_outputs)) + a
errors = []
for i in range(n_iterations):
# Calculate hidden layer
hidden_input = X_train.dot(self.W) + self.biasW
# Calculate output of hidden neurons
hidden_output = sigmoid(hidden_input)
# Calculate output layer
output_layer_input = hidden_output.dot(self.V) + self.biasV
output_layer_pred = sigmoid(output_layer_input)
# Calculate the error
error = y_train - output_layer_pred
mse = np.mean(np.power(error, 2))
errors.append(mse)
# Calculate loss gradients:
# Output layer weights V
v_gradient = -2*(y_train - output_layer_pred)*sigmoid_gradient(output_layer_input)
biasV_gradient = v_gradient
# Hidden layer weights W
w_gradient = v_gradient.dot(self.V.T)*sigmoid_gradient(hidden_input)
biasW_gradient = w_gradient
# Update weights
self.V -= learning_rate*hidden_output.T.dot(v_gradient)
self.biasV -= learning_rate*np.ones((1, n_samples)).dot(biasV_gradient)
self.W -= learning_rate*X_train.T.dot(w_gradient)
self.biasW -= learning_rate*np.ones((1, n_samples)).dot(biasW_gradient)
# Plot the training error
if plot_errors:
plt.plot(range(n_iterations), errors)
plt.ylabel('Training Error')
plt.xlabel('Iterations')
plt.show()
def fit(self, X, y):
y = categorical_to_binary(y)
y_pred = np.zeros(np.shape(y))
for i, tree in enumerate(self.trees):
y_and_pred = np.concatenate((y, y_pred), axis=1)
tree.fit(X, y_and_pred)
update_pred = tree.predict(X)
y_pred -= np.multiply(self.learning_rate, update_pred)
if self.debug:
progress = 100 * (i / self.n_estimators)
print("Progress: %.2f%%" % progress)
def fit(self, X, y):
y = categorical_to_binary(y)
# Set initial predictions to median of y
self.init_estimate = np.median(y, axis=0)
y_pred = np.zeros(np.shape(y))
for i, tree in enumerate(self.trees):
y_and_pred = np.concatenate((y, y_pred), axis=1)
tree.fit(X, y_and_pred)
update_pred = tree.predict(X)
y_pred += np.multiply(self.learning_rate, update_pred)
if self.debug:
progress = 100 * (i / self.n_estimators)
print ("Progress: %.2f%%" % progress)
def fit(self,
X,
y,
n_iterations=40000,
learning_rate=0.01,
plot_errors=False):
X_train = np.array(X, dtype=float)
y_train = categorical_to_binary(y)
n_outputs = np.shape(y_train)[1]
n_samples, n_features = np.shape(X_train)
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1 / math.sqrt(n_features)
b = -a
self.W = (b - a) * np.random.random((n_features, n_outputs)) + a
self.biasW = (b - a) * np.random.random((1, n_outputs)) + a
errors = []
for i in range(n_iterations):
# Calculate outputs
neuron_input = np.dot(X_train, self.W) + self.biasW
neuron_output = sigmoid(neuron_input)
error = y_train - neuron_output
mse = np.mean(np.power(error, 2))
errors.append(mse)
# Calculate the loss gradient
w_gradient = -2 * (y_train -
neuron_output) * sigmoid_gradient(neuron_input)
bias_gradient = w_gradient
# Update weights
self.W -= learning_rate * X_train.T.dot(w_gradient)
self.biasW -= learning_rate * np.ones(
(1, n_samples)).dot(bias_gradient)
# Plot the training error
if plot_errors:
plt.plot(range(n_iterations), errors)
plt.ylabel('Training Error')
plt.xlabel('Iterations')
plt.show()
def fit(self, X, y):
y = categorical_to_binary(y)
# Set initial predictions to median of y
self.init_estimate = np.median(y, axis=0)
y_pred = np.zeros(np.shape(y))
for i, tree in enumerate(self.trees):
gradient = self.loss.gradient(y, y_pred)
y_and_pred = np.concatenate((y, y_pred), axis=1)
tree.fit(X, y_and_pred)
gradient_est = tree.predict(X)
# Update y prediction by the gradient value
y_pred += np.multiply(self.learning_rate, gradient_est)
progress = 100 * (i / self.n_estimators)
if self.debug:
print ("Progress: %.2f%%" % progress)
def fit(self, X, y):
y = categorical_to_binary(y)
n_samples, n_features = np.shape(X)
n_outputs = np.shape(y)[1]
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1 / math.sqrt(n_features)
b = -a
self.W = (b - a) * np.random.random((n_features, n_outputs)) + a
self.biasW = (b - a) * np.random.random((1, n_outputs)) + a
errors = []
for i in range(self.n_iterations):
# Calculate outputs
neuron_input = np.dot(X, self.W) + self.biasW
neuron_output = sigmoid(neuron_input)
# Training error
error = y - neuron_output
mse = np.mean(np.power(error, 2))
errors.append(mse)
# Calculate the loss gradient
w_gradient = -2 * (y - neuron_output) * \
sigmoid_gradient(neuron_input)
bias_gradient = w_gradient
# Update weights
self.W -= self.learning_rate * X.T.dot(w_gradient)
self.biasW -= self.learning_rate * \
np.ones((1, n_samples)).dot(bias_gradient)
# Plot the training error
if self.plot_errors:
plt.plot(range(self.n_iterations), errors)
plt.ylabel('Training Error')
plt.xlabel('Iterations')
plt.title("Training Error Plot")
plt.show()
def fit(self, X, y):
y = categorical_to_binary(y)
super(GradientBoostingClassifier, self).fit(X, y)
def fit(self, X, y):
X_train = X
y_train = y
if self.early_stopping:
# Split the data into training and validation sets
X_train, X_validate, y_train, y_validate = train_test_split(
X, y, test_size=0.1)
y_validate = categorical_to_binary(y_validate)
# Convert the nominal y values to binary
y_train = categorical_to_binary(y_train)
n_samples, n_features = np.shape(X_train)
n_outputs = np.shape(y_train)[1]
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1 / math.sqrt(n_features)
b = -a
self.W = (b - a) * np.random.random((n_features, self.n_hidden)) + a
self.biasW = (b - a) * np.random.random((1, self.n_hidden)) + a
self.V = (b - a) * np.random.random((self.n_hidden, n_outputs)) + a
self.biasV = (b - a) * np.random.random((1, n_outputs)) + a
training_errors = []
validation_errors = []
iter_with_rising_val_error = 0
for i in range(self.n_iterations):
# Calculate hidden layer
hidden_input = X_train.dot(self.W) + self.biasW
hidden_output = sigmoid(hidden_input)
# Calculate output layer
output_layer_input = hidden_output.dot(self.V) + self.biasV
output = sigmoid(output_layer_input)
# Calculate the error
error = y_train - output
mse = np.mean(np.power(error, 2))
training_errors.append(mse)
# Calculate loss gradients:
# Gradient of squared loss w.r.t each parameter
v_gradient = -2 * (y_train - output) * \
sigmoid_gradient(output_layer_input)
biasV_gradient = v_gradient
w_gradient = v_gradient.dot(
self.V.T) * sigmoid_gradient(hidden_input)
biasW_gradient = w_gradient
# Update weights
# Move against the gradient to minimize loss
self.V -= self.learning_rate * hidden_output.T.dot(v_gradient)
self.biasV -= self.learning_rate * np.ones(
(1, n_samples)).dot(biasV_gradient)
self.W -= self.learning_rate * X_train.T.dot(w_gradient)
self.biasW -= self.learning_rate * np.ones(
(1, n_samples)).dot(biasW_gradient)
if self.early_stopping:
# Calculate the validation error
error = y_validate - self._calculate_output(X_validate)
mse = np.mean(np.power(error, 2))
validation_errors.append(mse)
# If the validation error is larger than the previous iteration increase
# the counter
if len(validation_errors
) > 1 and validation_errors[-1] > validation_errors[-2]:
iter_with_rising_val_error += 1
# # If the validation error has been for more than 50 iterations
# # stop training to avoid overfitting
if iter_with_rising_val_error > 50:
break
else:
iter_with_rising_val_error = 0
# Plot the training error
if self.plot_errors:
if self.early_stopping:
# Training and validation error plot
training, = plt.plot(range(i + 1),
training_errors,
label="Training Error")
validation, = plt.plot(range(i + 1),
validation_errors,
label="Validation Error")
plt.legend(handles=[training, validation])
else:
training, = plt.plot(range(i + 1),
training_errors,
label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
def fit(self, X, y):
X_train = X
y_train = y
if self.early_stopping:
# Split the data into training and validation sets
X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.1)
y_validate = categorical_to_binary(y_validate)
# Convert the nominal y values to binary
y_train = categorical_to_binary(y_train)
n_samples, n_features = np.shape(X_train)
n_outputs = np.shape(y_train)[1]
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1 / math.sqrt(n_features)
b = -a
self.W = (b - a) * np.random.random((n_features, n_outputs)) + a
self.biasW = (b - a) * np.random.random((1, n_outputs)) + a
training_errors = []
validation_errors = []
iter_with_rising_val_error = 0
for i in range(self.n_iterations):
# Calculate outputs
neuron_input = np.dot(X_train, self.W) + self.biasW
neuron_output = sigmoid(neuron_input)
# Training error
error = y_train - neuron_output
mse = np.mean(np.power(error, 2))
training_errors.append(mse)
# Calculate the loss gradient
w_gradient = -2 * (y_train - neuron_output) * \
sigmoid_gradient(neuron_input)
bias_gradient = w_gradient
# Update weights
self.W -= self.learning_rate * X_train.T.dot(w_gradient)
self.biasW -= self.learning_rate * \
np.ones((1, n_samples)).dot(bias_gradient)
if self.early_stopping:
# Calculate the validation error
error = y_validate - self._calculate_output(X_validate)
mse = np.mean(np.power(error, 2))
validation_errors.append(mse)
# If the validation error is larger than the previous iteration increase
# the counter
if len(validation_errors) > 1 and validation_errors[-1] > validation_errors[-2]:
iter_with_rising_val_error += 1
# If the validation error has been for more than 50 iterations
# stop training to avoid overfitting
if iter_with_rising_val_error > 50:
break
else:
iter_with_rising_val_error = 0
# Plot the training error
if self.plot_errors:
if self.early_stopping:
# Training and validation error plot
training, = plt.plot(range(i+1), training_errors, label="Training Error")
validation, = plt.plot(range(i+1), validation_errors, label="Validation Error")
plt.legend(handles=[training, validation])
else:
training, = plt.plot(range(i+1), training_errors, label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
def fit(self, X, y):
y = categorical_to_binary(y)
super(GradientBoostingClassifier, self).fit(X, y)
def fit(self, X, y):
X_train = X
y_train = y
if self.early_stopping:
# Split the data into training and validation sets
X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.1)
y_validate = categorical_to_binary(y_validate)
# Convert the nominal y values to binary
y_train = categorical_to_binary(y_train)
n_samples, n_features = np.shape(X_train)
n_outputs = np.shape(y_train)[1]
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1 / math.sqrt(n_features)
b = -a
self.W = (b - a) * np.random.random((n_features, n_outputs)) + a
self.biasW = (b - a) * np.random.random((1, n_outputs)) + a
training_errors = []
validation_errors = []
iter_with_rising_val_error = 0
w_updt = np.zeros(np.shape(self.W))
b_updt = np.zeros(np.shape(self.biasW))
for i in range(self.n_iterations):
# Calculate outputs
neuron_input = np.dot(X_train, self.W) + self.biasW
neuron_output = self.activation.function(neuron_input)
# Training error
error = y_train - neuron_output
mse = np.mean(np.power(error, 2))
training_errors.append(mse)
# Calculate the loss gradient
w_gradient = -2 * (y_train - neuron_output) * \
self.activation.gradient(neuron_input)
bias_gradient = w_gradient
# Calculate the weight updates
w_updt = self.momentum*w_updt + X_train.T.dot(w_gradient)
b_updt = self.momentum*b_updt + np.ones((1, n_samples)).dot(bias_gradient)
# Update weights
self.W -= self.learning_rate * w_updt
self.biasW -= self.learning_rate * b_updt
if self.early_stopping:
# Calculate the validation error
error = y_validate - self._calculate_output(X_validate)
mse = np.mean(np.power(error, 2))
validation_errors.append(mse)
# If the validation error is larger than the previous iteration increase
# the counter
if len(validation_errors) > 1 and validation_errors[-1] > validation_errors[-2]:
iter_with_rising_val_error += 1
# If the validation error has been for more than 50 iterations
# stop training to avoid overfitting
if iter_with_rising_val_error > 50:
break
else:
iter_with_rising_val_error = 0
# Plot the training error
if self.plot_errors:
if self.early_stopping:
# Training and validation error plot
training, = plt.plot(range(i+1), training_errors, label="Training Error")
validation, = plt.plot(range(i+1), validation_errors, label="Validation Error")
plt.legend(handles=[training, validation])
else:
training, = plt.plot(range(i+1), training_errors, label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
def fit(self, X, y):
X_train = X
y_train = y
if self.early_stopping:
# Split the data into training and validation sets
X_train, X_validate, y_train, y_validate = train_test_split(
X, y, test_size=0.1)
y_validate = categorical_to_binary(y_validate)
# Convert the nominal y values to binary
y_train = categorical_to_binary(y_train)
n_samples, n_features = np.shape(X_train)
n_outputs = np.shape(y_train)[1]
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1 / math.sqrt(n_features)
b = -a
self.W = (b - a) * np.random.random((n_features, self.n_hidden)) + a
self.biasW = (b - a) * np.random.random((1, self.n_hidden)) + a
self.V = (b - a) * np.random.random((self.n_hidden, n_outputs)) + a
self.biasV = (b - a) * np.random.random((1, n_outputs)) + a
# Error history
training_errors = []
validation_errors = []
iter_with_rising_val_error = 0
for i in range(self.n_iterations):
# Calculate hidden layer
hidden_input = X_train.dot(self.W) + self.biasW
# hidden_output = sigmoid(hidden_input)
hidden_output = self.activation.function(hidden_input)
# Calculate output layer
output_layer_input = hidden_output.dot(self.V) + self.biasV
# output = sigmoid(output_layer_input)
output = self.activation.function(output_layer_input)
# Calculate the error
error = y_train - output
mse = np.mean(np.power(error, 2))
training_errors.append(mse)
# Calculate the loss gradient
output_gradient = -2 * (y_train - output) * \
self.activation.gradient(output_layer_input)
hidden_gradient = output_gradient.dot(
self.V.T) * self.activation.gradient(hidden_input)
# Calcualte the gradient with respect to each weight term
grad_wrt_v = hidden_output.T.dot(output_gradient)
grad_wrt_vb = np.ones((1, n_samples)).dot(output_gradient)
grad_wrt_w = X_train.T.dot(hidden_gradient)
grad_wrt_wb = np.ones((1, n_samples)).dot(hidden_gradient)
# Update weights
# Move against the gradient to minimize loss
self.V = self.v_opt.update(w=self.V, grad_wrt_w=grad_wrt_v)
self.biasV = self.vb_opt.update(w=self.biasV,
grad_wrt_w=grad_wrt_vb)
self.W = self.w_opt.update(w=self.W, grad_wrt_w=grad_wrt_w)
self.biasW = self.wb_opt.update(w=self.biasW,
grad_wrt_w=grad_wrt_wb)
if self.early_stopping:
# Calculate the validation error
error = y_validate - self._calculate_output(X_validate)
mse = np.mean(np.power(error, 2))
validation_errors.append(mse)
# If the validation error is larger than the previous iteration increase
# the counter
if len(validation_errors
) > 1 and validation_errors[-1] > validation_errors[-2]:
iter_with_rising_val_error += 1
# # If the validation error has been for more than 50 iterations
# # stop training to avoid overfitting
if iter_with_rising_val_error > 50:
break
else:
iter_with_rising_val_error = 0
# Plot the training error
if self.plot_errors:
if self.early_stopping:
# Training and validation error plot
training, = plt.plot(range(i + 1),
training_errors,
label="Training Error")
validation, = plt.plot(range(i + 1),
validation_errors,
label="Validation Error")
plt.legend(handles=[training, validation])
else:
training, = plt.plot(range(i + 1),
training_errors,
label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
def fit(self, X, y):
X_train = X
y_train = y
if self.early_stopping:
# Split the data into training and validation sets
X_train, X_validate, y_train, y_validate = train_test_split(
X, y, test_size=0.1)
y_validate = categorical_to_binary(y_validate)
# Convert the nominal y values to binary
y_train = categorical_to_binary(y_train)
n_samples, n_features = np.shape(X_train)
n_outputs = np.shape(y_train)[1]
# Initial weights between [-1/sqrt(N), 1/sqrt(N)]
a = -1 / math.sqrt(n_features)
b = -a
self.W = (b - a) * np.random.random((n_features, n_outputs)) + a
self.biasW = (b - a) * np.random.random((1, n_outputs)) + a
# Error history
training_errors = []
validation_errors = []
iter_with_rising_val_error = 0
# Lambda function that calculates the neuron outputs
neuron_output = lambda w, b: self.activation.function(
np.dot(X_train, w) + b)
# Lambda function that calculates the loss gradient
loss_grad = lambda w, b: -2 * (y_train - neuron_output(w, b)) * \
self.activation.gradient(np.dot(X_train, w) + b)
# Lambda functions that calculates the gradient of the loss with
# respect to each weight term. Allows for computation of loss gradient at different
# coordinates.
grad_func_wrt_w = lambda w: X_train.T.dot(loss_grad(w, self.biasW))
grad_func_wrt_bias = lambda b: np.ones(
(1, n_samples)).dot(loss_grad(self.W, b))
# Optimize paramaters for n_iterations
for i in range(self.n_iterations):
# Training error
error = y_train - neuron_output(self.W, self.biasW)
mse = np.mean(np.power(error, 2))
training_errors.append(mse)
# Update weights
self.W = self.w_opt.update(w=self.W, grad_func=grad_func_wrt_w)
self.biasW = self.bias_opt.update(w=self.biasW,
grad_func=grad_func_wrt_bias)
if self.early_stopping:
# Calculate the validation error
error = y_validate - self._calculate_output(X_validate)
mse = np.mean(np.power(error, 2))
validation_errors.append(mse)
# If the validation error is larger than the previous iteration increase
# the counter
if len(validation_errors
) > 1 and validation_errors[-1] > validation_errors[-2]:
iter_with_rising_val_error += 1
# If the validation error has been for more than 50 iterations
# stop training to avoid overfitting
if iter_with_rising_val_error > 50:
break
else:
iter_with_rising_val_error = 0
# Plot the training error
if self.plot_errors:
if self.early_stopping:
# Training and validation error plot
training, = plt.plot(range(i + 1),
training_errors,
label="Training Error")
validation, = plt.plot(range(i + 1),
validation_errors,
label="Validation Error")
plt.legend(handles=[training, validation])
else:
training, = plt.plot(range(i + 1),
training_errors,
label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
