def fit(self, X, y, n_iterations=3000, learning_rate=0.01, plot_errors=False): # Convert the nominal y values to binary y = categorical_to_binary(y) n_samples, n_features = np.shape(X) n_outputs = np.shape(y)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, self.n_hidden)) + a self.biasW = (b - a) * np.random.random((1, self.n_hidden)) + a self.V = (b - a) * np.random.random((self.n_hidden, n_outputs)) + a self.biasV = (b - a) * np.random.random((1, n_outputs)) + a errors = [] for i in range(n_iterations): # Calculate hidden layer hidden_input = X.dot(self.W) + self.biasW # Calculate output of hidden neurons hidden_output = sigmoid(hidden_input) # Calculate output layer output_layer_input = hidden_output.dot(self.V) + self.biasV output_layer_pred = sigmoid(output_layer_input) # Calculate the error error = y - output_layer_pred mse = np.mean(np.power(error, 2)) errors.append(mse) # Calculate loss gradients: # Gradient of squared loss w.r.t each parameter v_gradient = -2 * (y - output_layer_pred) * \ sigmoid_gradient(output_layer_input) biasV_gradient = v_gradient w_gradient = v_gradient.dot( self.V.T) * sigmoid_gradient(hidden_input) biasW_gradient = w_gradient # Update weights # Move against the gradient to minimize loss self.V -= learning_rate * hidden_output.T.dot(v_gradient) self.biasV -= learning_rate * np.ones( (1, n_samples)).dot(biasV_gradient) self.W -= learning_rate * X.T.dot(w_gradient) self.biasW -= learning_rate * np.ones( (1, n_samples)).dot(biasW_gradient) # Plot the training error if plot_errors: plt.plot(range(n_iterations), errors) plt.ylabel('Training Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y, n_iterations=3000, learning_rate=0.01, plot_errors=False): X_train = np.array(X, dtype=float) # Convert the nominal y values to binary y_train = categorical_to_binary(y) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1/math.sqrt(n_features) b = -a self.W = (b-a)*np.random.random((n_features, self.n_hidden)) + a self.biasW = (b-a)*np.random.random((1, self.n_hidden)) + a self.V = (b-a)*np.random.random((self.n_hidden, n_outputs)) + a self.biasV = (b-a)*np.random.random((1, n_outputs)) + a errors = [] for i in range(n_iterations): # Calculate hidden layer hidden_input = X_train.dot(self.W) + self.biasW # Calculate output of hidden neurons hidden_output = sigmoid(hidden_input) # Calculate output layer output_layer_input = hidden_output.dot(self.V) + self.biasV output_layer_pred = sigmoid(output_layer_input) # Calculate the error error = y_train - output_layer_pred mse = np.mean(np.power(error, 2)) errors.append(mse) # Calculate loss gradients: # Output layer weights V v_gradient = -2*(y_train - output_layer_pred)*sigmoid_gradient(output_layer_input) biasV_gradient = v_gradient # Hidden layer weights W w_gradient = v_gradient.dot(self.V.T)*sigmoid_gradient(hidden_input) biasW_gradient = w_gradient # Update weights self.V -= learning_rate*hidden_output.T.dot(v_gradient) self.biasV -= learning_rate*np.ones((1, n_samples)).dot(biasV_gradient) self.W -= learning_rate*X_train.T.dot(w_gradient) self.biasW -= learning_rate*np.ones((1, n_samples)).dot(biasW_gradient) # Plot the training error if plot_errors: plt.plot(range(n_iterations), errors) plt.ylabel('Training Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): y = categorical_to_binary(y) y_pred = np.zeros(np.shape(y)) for i, tree in enumerate(self.trees): y_and_pred = np.concatenate((y, y_pred), axis=1) tree.fit(X, y_and_pred) update_pred = tree.predict(X) y_pred -= np.multiply(self.learning_rate, update_pred) if self.debug: progress = 100 * (i / self.n_estimators) print("Progress: %.2f%%" % progress)
def fit(self, X, y): y = categorical_to_binary(y) # Set initial predictions to median of y self.init_estimate = np.median(y, axis=0) y_pred = np.zeros(np.shape(y)) for i, tree in enumerate(self.trees): y_and_pred = np.concatenate((y, y_pred), axis=1) tree.fit(X, y_and_pred) update_pred = tree.predict(X) y_pred += np.multiply(self.learning_rate, update_pred) if self.debug: progress = 100 * (i / self.n_estimators) print ("Progress: %.2f%%" % progress)
def fit(self, X, y, n_iterations=40000, learning_rate=0.01, plot_errors=False): X_train = np.array(X, dtype=float) y_train = categorical_to_binary(y) n_outputs = np.shape(y_train)[1] n_samples, n_features = np.shape(X_train) # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.biasW = (b - a) * np.random.random((1, n_outputs)) + a errors = [] for i in range(n_iterations): # Calculate outputs neuron_input = np.dot(X_train, self.W) + self.biasW neuron_output = sigmoid(neuron_input) error = y_train - neuron_output mse = np.mean(np.power(error, 2)) errors.append(mse) # Calculate the loss gradient w_gradient = -2 * (y_train - neuron_output) * sigmoid_gradient(neuron_input) bias_gradient = w_gradient # Update weights self.W -= learning_rate * X_train.T.dot(w_gradient) self.biasW -= learning_rate * np.ones( (1, n_samples)).dot(bias_gradient) # Plot the training error if plot_errors: plt.plot(range(n_iterations), errors) plt.ylabel('Training Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): y = categorical_to_binary(y) # Set initial predictions to median of y self.init_estimate = np.median(y, axis=0) y_pred = np.zeros(np.shape(y)) for i, tree in enumerate(self.trees): gradient = self.loss.gradient(y, y_pred) y_and_pred = np.concatenate((y, y_pred), axis=1) tree.fit(X, y_and_pred) gradient_est = tree.predict(X) # Update y prediction by the gradient value y_pred += np.multiply(self.learning_rate, gradient_est) progress = 100 * (i / self.n_estimators) if self.debug: print ("Progress: %.2f%%" % progress)
def fit(self, X, y): y = categorical_to_binary(y) n_samples, n_features = np.shape(X) n_outputs = np.shape(y)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.biasW = (b - a) * np.random.random((1, n_outputs)) + a errors = [] for i in range(self.n_iterations): # Calculate outputs neuron_input = np.dot(X, self.W) + self.biasW neuron_output = sigmoid(neuron_input) # Training error error = y - neuron_output mse = np.mean(np.power(error, 2)) errors.append(mse) # Calculate the loss gradient w_gradient = -2 * (y - neuron_output) * \ sigmoid_gradient(neuron_input) bias_gradient = w_gradient # Update weights self.W -= self.learning_rate * X.T.dot(w_gradient) self.biasW -= self.learning_rate * \ np.ones((1, n_samples)).dot(bias_gradient) # Plot the training error if self.plot_errors: plt.plot(range(self.n_iterations), errors) plt.ylabel('Training Error') plt.xlabel('Iterations') plt.title("Training Error Plot") plt.show()
def fit(self, X, y): y = categorical_to_binary(y) super(GradientBoostingClassifier, self).fit(X, y)
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split( X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, self.n_hidden)) + a self.biasW = (b - a) * np.random.random((1, self.n_hidden)) + a self.V = (b - a) * np.random.random((self.n_hidden, n_outputs)) + a self.biasV = (b - a) * np.random.random((1, n_outputs)) + a training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 for i in range(self.n_iterations): # Calculate hidden layer hidden_input = X_train.dot(self.W) + self.biasW hidden_output = sigmoid(hidden_input) # Calculate output layer output_layer_input = hidden_output.dot(self.V) + self.biasV output = sigmoid(output_layer_input) # Calculate the error error = y_train - output mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Calculate loss gradients: # Gradient of squared loss w.r.t each parameter v_gradient = -2 * (y_train - output) * \ sigmoid_gradient(output_layer_input) biasV_gradient = v_gradient w_gradient = v_gradient.dot( self.V.T) * sigmoid_gradient(hidden_input) biasW_gradient = w_gradient # Update weights # Move against the gradient to minimize loss self.V -= self.learning_rate * hidden_output.T.dot(v_gradient) self.biasV -= self.learning_rate * np.ones( (1, n_samples)).dot(biasV_gradient) self.W -= self.learning_rate * X_train.T.dot(w_gradient) self.biasW -= self.learning_rate * np.ones( (1, n_samples)).dot(biasW_gradient) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors ) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # # If the validation error has been for more than 50 iterations # # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i + 1), training_errors, label="Training Error") validation, = plt.plot(range(i + 1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i + 1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.biasW = (b - a) * np.random.random((1, n_outputs)) + a training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 for i in range(self.n_iterations): # Calculate outputs neuron_input = np.dot(X_train, self.W) + self.biasW neuron_output = sigmoid(neuron_input) # Training error error = y_train - neuron_output mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Calculate the loss gradient w_gradient = -2 * (y_train - neuron_output) * \ sigmoid_gradient(neuron_input) bias_gradient = w_gradient # Update weights self.W -= self.learning_rate * X_train.T.dot(w_gradient) self.biasW -= self.learning_rate * \ np.ones((1, n_samples)).dot(bias_gradient) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # If the validation error has been for more than 50 iterations # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i+1), training_errors, label="Training Error") validation, = plt.plot(range(i+1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i+1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): y = categorical_to_binary(y) super(GradientBoostingClassifier, self).fit(X, y)
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.biasW = (b - a) * np.random.random((1, n_outputs)) + a training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 w_updt = np.zeros(np.shape(self.W)) b_updt = np.zeros(np.shape(self.biasW)) for i in range(self.n_iterations): # Calculate outputs neuron_input = np.dot(X_train, self.W) + self.biasW neuron_output = self.activation.function(neuron_input) # Training error error = y_train - neuron_output mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Calculate the loss gradient w_gradient = -2 * (y_train - neuron_output) * \ self.activation.gradient(neuron_input) bias_gradient = w_gradient # Calculate the weight updates w_updt = self.momentum*w_updt + X_train.T.dot(w_gradient) b_updt = self.momentum*b_updt + np.ones((1, n_samples)).dot(bias_gradient) # Update weights self.W -= self.learning_rate * w_updt self.biasW -= self.learning_rate * b_updt if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # If the validation error has been for more than 50 iterations # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i+1), training_errors, label="Training Error") validation, = plt.plot(range(i+1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i+1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split( X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, self.n_hidden)) + a self.biasW = (b - a) * np.random.random((1, self.n_hidden)) + a self.V = (b - a) * np.random.random((self.n_hidden, n_outputs)) + a self.biasV = (b - a) * np.random.random((1, n_outputs)) + a # Error history training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 for i in range(self.n_iterations): # Calculate hidden layer hidden_input = X_train.dot(self.W) + self.biasW # hidden_output = sigmoid(hidden_input) hidden_output = self.activation.function(hidden_input) # Calculate output layer output_layer_input = hidden_output.dot(self.V) + self.biasV # output = sigmoid(output_layer_input) output = self.activation.function(output_layer_input) # Calculate the error error = y_train - output mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Calculate the loss gradient output_gradient = -2 * (y_train - output) * \ self.activation.gradient(output_layer_input) hidden_gradient = output_gradient.dot( self.V.T) * self.activation.gradient(hidden_input) # Calcualte the gradient with respect to each weight term grad_wrt_v = hidden_output.T.dot(output_gradient) grad_wrt_vb = np.ones((1, n_samples)).dot(output_gradient) grad_wrt_w = X_train.T.dot(hidden_gradient) grad_wrt_wb = np.ones((1, n_samples)).dot(hidden_gradient) # Update weights # Move against the gradient to minimize loss self.V = self.v_opt.update(w=self.V, grad_wrt_w=grad_wrt_v) self.biasV = self.vb_opt.update(w=self.biasV, grad_wrt_w=grad_wrt_vb) self.W = self.w_opt.update(w=self.W, grad_wrt_w=grad_wrt_w) self.biasW = self.wb_opt.update(w=self.biasW, grad_wrt_w=grad_wrt_wb) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors ) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # # If the validation error has been for more than 50 iterations # # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i + 1), training_errors, label="Training Error") validation, = plt.plot(range(i + 1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i + 1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split( X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.biasW = (b - a) * np.random.random((1, n_outputs)) + a # Error history training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 # Lambda function that calculates the neuron outputs neuron_output = lambda w, b: self.activation.function( np.dot(X_train, w) + b) # Lambda function that calculates the loss gradient loss_grad = lambda w, b: -2 * (y_train - neuron_output(w, b)) * \ self.activation.gradient(np.dot(X_train, w) + b) # Lambda functions that calculates the gradient of the loss with # respect to each weight term. Allows for computation of loss gradient at different # coordinates. grad_func_wrt_w = lambda w: X_train.T.dot(loss_grad(w, self.biasW)) grad_func_wrt_bias = lambda b: np.ones( (1, n_samples)).dot(loss_grad(self.W, b)) # Optimize paramaters for n_iterations for i in range(self.n_iterations): # Training error error = y_train - neuron_output(self.W, self.biasW) mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Update weights self.W = self.w_opt.update(w=self.W, grad_func=grad_func_wrt_w) self.biasW = self.bias_opt.update(w=self.biasW, grad_func=grad_func_wrt_bias) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors ) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # If the validation error has been for more than 50 iterations # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i + 1), training_errors, label="Training Error") validation, = plt.plot(range(i + 1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i + 1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()