def fit(self, X, y): X_train = X y_train = y if self.val_error: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split( X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_batches = int(n_samples / self.batch_size) bar = progressbar.ProgressBar(widgets=bar_widgets) for i in bar(range(self.n_iterations)): X_, y_ = shuffle_data(X_train, y_train) batch_t_error = 0 # Mean batch training error batch_v_error = 0 # Mean batch validation error for idx in np.array_split(np.arange(n_samples), n_batches): X_batch, y_batch = X_[idx], y_[idx] # Calculate output y_pred = self._forward_pass(X_batch) # Calculate the cross entropy training loss loss = np.mean(self.cross_ent.loss(y_batch, y_pred)) batch_t_error += loss loss_grad = self.cross_ent.gradient(y_batch, y_pred) # Update the NN weights self._backward_pass(loss_grad=loss_grad) if self.val_error: # Calculate the validation error y_val_pred = self._forward_pass(X_validate) loss = np.mean(self.cross_ent.loss(y_validate, y_val_pred)) batch_v_error += loss batch_t_error /= n_batches batch_v_error /= n_batches self.errors["training"].append(batch_t_error) self.errors["validation"].append(batch_v_error)
def fit(self, X, y): # Convert the categorical data to binary y = categorical_to_binary(y.astype("int")) n_samples, n_features = np.shape(X) n_batches = int(n_samples / self.batch_size) bar = progressbar.ProgressBar(widgets=bar_widgets) for _ in bar(range(self.n_iterations)): X_, y_ = shuffle_data(X, y) batch_t_error = 0 # Mean batch training error for idx in np.array_split(np.arange(n_samples), n_batches): X_batch, y_batch = X_[idx], y_[idx] # Calculate output y_pred = self._forward_pass(X_batch) # Calculate the cross entropy training loss loss = np.mean(self.cross_ent.loss(y_batch, y_pred)) batch_t_error += loss loss_grad = self.cross_ent.gradient(y_batch, y_pred) # Update the NN weights self._backward_pass(loss_grad=loss_grad) batch_t_error /= n_batches self.errors["training"].append(batch_t_error) if self.X_val.any(): # Calculate the validation error y_val_p = self._forward_pass(self.X_val) loss = np.mean(self.cross_ent.loss(self.y_val, y_val_p)) self.errors["validation"].append(loss)
def fit(self, X, y): y = categorical_to_binary(y) y_pred = np.zeros(np.shape(y)) for i in self.bar(range(self.n_estimators)): tree = self.trees[i] y_and_pred = np.concatenate((y, y_pred), axis=1) tree.fit(X, y_and_pred) update_pred = tree.predict(X) y_pred -= np.multiply(self.learning_rate, update_pred)
def fit(self, X, y): y = categorical_to_binary(y) y_pred = np.zeros(np.shape(y)) for i, tree in enumerate(self.trees): y_and_pred = np.concatenate((y, y_pred), axis=1) tree.fit(X, y_and_pred) update_pred = tree.predict(X) y_pred -= np.multiply(self.learning_rate, update_pred) if self.debug: progress = 100 * (i / self.n_estimators) print("Progress: %.2f%%" % progress)
def __init__(self, n_iterations, batch_size, optimizer, loss, validation_data=None): self.n_iterations = n_iterations self.optimizer = optimizer self.layers = [] self.errors = {"training": [], "validation": []} self.cross_ent = loss() self.batch_size = batch_size self.X_val = self.y_val = np.empty([]) if validation_data: self.X_val, self.y_val = validation_data self.y_val = categorical_to_binary(self.y_val.astype("int"))
def fit(self, X, y): # Convert the categorical data to binary y = categorical_to_binary(y.astype("int")) n_samples = np.shape(X)[0] n_batches = int(n_samples / self.batch_size) bar = progressbar.ProgressBar(widgets=bar_widgets) for _ in bar(range(self.n_iterations)): idx = range(n_samples) np.random.shuffle(idx) batch_t_error = 0 # Mean batch training error for i in range(n_batches): X_batch = X[idx[i * self.batch_size:(i + 1) * self.batch_size]] y_batch = y[idx[i * self.batch_size:(i + 1) * self.batch_size]] # Calculate output y_pred = self._forward_pass(X_batch) # Calculate the cross entropy training loss loss = np.mean(self.loss_function.loss(y_batch, y_pred)) batch_t_error += loss loss_grad = self.loss_function.gradient(y_batch, y_pred) # Backprop. Update weights self._backward_pass(loss_grad=loss_grad) # Save the epoch mean error self.errors["training"].append(batch_t_error / n_batches) if self.X_val.any(): # Calculate the validation error y_val_p = self._forward_pass(self.X_val) validation_loss = np.mean( self.loss_function.loss(self.y_val, y_val_p)) self.errors["validation"].append(validation_loss)
def fit(self, X, y): y = categorical_to_binary(y) super(GradientBoostingClassifier, self).fit(X, y)
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, self.n_hidden)) + a self.w0 = (b - a) * np.random.random((1, self.n_hidden)) + a self.V = (b - a) * np.random.random((self.n_hidden, n_outputs)) + a self.v0 = (b - a) * np.random.random((1, n_outputs)) + a # Error history training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 for i in range(self.n_iterations): # Calculate hidden layer hidden_layer_input = X_train.dot(self.W) + self.w0 hidden_layer_output = self.activation.function(hidden_layer_input) # Calculate output layer output_layer_input = hidden_layer_output.dot(self.V) + self.v0 output = self.activation.function(output_layer_input) # Calculate the error error = y_train - output mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Calculate the loss gradient output_gradient = -2 * (y_train - output) * \ self.activation.gradient(output_layer_input) hidden_gradient = output_gradient.dot( self.V.T) * self.activation.gradient(hidden_layer_input) # Calcualte the gradient with respect to each weight term grad_wrt_v = hidden_layer_output.T.dot(output_gradient) grad_wrt_v0 = np.ones((1, n_samples)).dot(output_gradient) grad_wrt_w = X_train.T.dot(hidden_gradient) grad_wrt_w0 = np.ones((1, n_samples)).dot(hidden_gradient) # Update weights # Move against the gradient to minimize loss self.V = self.v_opt.update(w=self.V, grad_wrt_w=grad_wrt_v) self.v0 = self.v0_opt.update(w=self.v0, grad_wrt_w=grad_wrt_v0) self.W = self.w_opt.update(w=self.W, grad_wrt_w=grad_wrt_w) self.w0 = self.w0_opt.update(w=self.w0, grad_wrt_w=grad_wrt_w0) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # # If the validation error has been for more than 50 iterations # # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i+1), training_errors, label="Training Error") validation, = plt.plot(range(i+1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i+1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.w0 = (b - a) * np.random.random((1, n_outputs)) + a # Error history training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 # Lambda function that calculates the neuron outputs neuron_output = lambda w, b: self.activation.function(np.dot(X_train, w) + b) # Lambda function that calculates the loss gradient loss_grad = lambda w, b: -2 * (y_train - neuron_output(w, b)) * \ self.activation.gradient(np.dot(X_train, w) + b) # Lambda functions that calculates the gradient of the loss with # respect to each weight term. Allows for computation of loss gradient at different # coordinates. grad_func_wrt_w = lambda w: X_train.T.dot(loss_grad(w, self.w0)) grad_func_wrt_w0 = lambda b: np.ones((1, n_samples)).dot(loss_grad(self.W, b)) # Optimize paramaters for n_iterations for i in range(self.n_iterations): # Training error error = y_train - neuron_output(self.W, self.w0) mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Update weights self.W = self.w_opt.update(w=self.W, grad_func=grad_func_wrt_w) self.w0 = self.w0_opt.update(w=self.w0, grad_func=grad_func_wrt_w0) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # If the validation error has been for more than 50 iterations # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i+1), training_errors, label="Training Error") validation, = plt.plot(range(i+1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i+1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split( X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.w0 = (b - a) * np.random.random((1, n_outputs)) + a # Error history training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 # Lambda function that calculates the neuron outputs neuron_output = lambda w, b: self.activation.function( np.dot(X_train, w) + b) # Lambda function that calculates the loss gradient loss_grad = lambda w, b: -2 * (y_train - neuron_output(w, b)) * \ self.activation.gradient(np.dot(X_train, w) + b) # Lambda functions that calculates the gradient of the loss with # respect to each weight term. Allows for computation of loss gradient at different # coordinates. grad_func_wrt_w = lambda w: X_train.T.dot(loss_grad(w, self.w0)) grad_func_wrt_w0 = lambda b: np.ones( (1, n_samples)).dot(loss_grad(self.W, b)) # Optimize paramaters for n_iterations for i in range(self.n_iterations): # Training error error = y_train - neuron_output(self.W, self.w0) mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Update weights self.W = self.w_opt.update(w=self.W, grad_func=grad_func_wrt_w) self.w0 = self.w0_opt.update(w=self.w0, grad_func=grad_func_wrt_w0) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors ) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # If the validation error has been for more than 50 iterations # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i + 1), training_errors, label="Training Error") validation, = plt.plot(range(i + 1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i + 1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()