def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target # Convert the nominal y values to binary y = to_categorical(y) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) # MLP clf = MultilayerPerceptron(n_hidden=12, n_iterations=1000, learning_rate=0.01) clf.fit(X_train, y_train) y_pred = np.argmax(clf.predict(X_test), axis=1) y_test = np.argmax(y_test, axis=1) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Multilayer Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def fit(self, X, y): X_train = X y_train = y # One-hot encoding of nominal y-values y_train = to_categorical(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initialize weights between [-1/sqrt(N), 1/sqrt(N)] limit = -1 / math.sqrt(n_features) self.W = np.random.uniform(-limit, limit, (n_features, n_outputs)) self.w0 = np.zeros((1, n_outputs)) for i in range(self.n_iterations): # Calculate outputs linear_output = np.dot(X_train, self.W) + self.w0 y_pred = self.activation.function(linear_output) # Calculate the loss gradient error_gradient = -2 * (y_train - y_pred) * \ self.activation.gradient(linear_output) # Calculate the gradient of the loss with respect to each weight term grad_wrt_w = X_train.T.dot(error_gradient) grad_wrt_w0 = np.ones((1, n_samples)).dot(error_gradient) # Update weights self.W -= self.learning_rate * grad_wrt_w self.w0 -= self.learning_rate * grad_wrt_w0
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target # One-hot encoding of nominal y-values y = to_categorical(y) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) # Perceptron clf = Perceptron(n_iterations=5000, learning_rate=0.001, activation_function=Sigmoid) clf.fit(X_train, y_train) y_pred = np.argmax(clf.predict(X_test), axis=1) y_test = np.argmax(y_test, axis=1) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def fit(self, X, y, n_epochs, batch_size): # Convert to one-hot encoding y = to_categorical(y.astype("int")) n_samples = np.shape(X)[0] n_batches = int(n_samples / batch_size) bar = progressbar.ProgressBar(widgets=bar_widgets) for _ in bar(range(n_epochs)): idx = range(n_samples) np.random.shuffle(idx) batch_t_error = 0 # Mean batch training error for i in range(n_batches): X_batch = X[idx[i * batch_size:(i + 1) * batch_size]] y_batch = y[idx[i * batch_size:(i + 1) * batch_size]] loss, _ = self.train_on_batch(X_batch, y_batch) batch_t_error += loss # Save the epoch mean error self.errors["training"].append(batch_t_error / n_batches) if self.X_val.any(): # Calculate the validation error y_val_p = self._forward_pass(self.X_val) validation_loss = np.mean( self.loss_function.loss(self.y_val, y_val_p)) self.errors["validation"].append(validation_loss) return self.errors["training"], self.errors["validation"]
def fit(self, X, y): X_train = X y_train = y # One-hot encoding of nominal y-values y_train = to_categorical(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initialize weights between [-1/sqrt(N), 1/sqrt(N)] limit = 1 / math.sqrt(n_features) self.W = np.random.uniform(-limit, limit, (n_features, n_outputs)) self.w0 = np.zeros((1, n_outputs)) for i in range(self.n_iterations): # Calculate outputs linear_output = np.dot(X_train, self.W) + self.w0 y_pred = self.activation.function(linear_output) # Calculate the loss gradient error_gradient = -2 * (y_train - y_pred) * \ self.activation.gradient(linear_output) # Calculate the gradient of the loss with respect to each weight term grad_wrt_w = X_train.T.dot(error_gradient) grad_wrt_w0 = np.ones((1, n_samples)).dot(error_gradient) # Update weights self.W -= self.learning_rate * grad_wrt_w self.w0 -= self.learning_rate * grad_wrt_w0
def fit(self, X, y, n_iterations, batch_size): # Convert to one-hot encoding y = to_categorical(y.astype("int")) n_samples = np.shape(X)[0] n_batches = int(n_samples / batch_size) bar = progressbar.ProgressBar(widgets=bar_widgets) for _ in bar(range(n_iterations)): idx = range(n_samples) np.random.shuffle(idx) batch_t_error = 0 # Mean batch training error for i in range(n_batches): X_batch = X[idx[i*batch_size:(i+1)*batch_size]] y_batch = y[idx[i*batch_size:(i+1)*batch_size]] loss, _ = self.train_on_batch(X_batch, y_batch) batch_t_error += loss # Save the epoch mean error self.errors["training"].append(batch_t_error / n_batches) if self.X_val.any(): # Calculate the validation error y_val_p = self._forward_pass(self.X_val) validation_loss = np.mean(self.loss_function.loss(self.y_val, y_val_p)) self.errors["validation"].append(validation_loss)
def __init__(self, optimizer, loss, validation_data=None): self.optimizer = optimizer self.layers = [] self.errors = {"training": [], "validation": []} self.loss_function = loss() self.X_val = np.empty([]) self.y_val = np.empty([]) if validation_data: self.X_val, self.y_val = validation_data self.y_val = to_categorical(self.y_val.astype("int"))
def __init__(self, optimizer, loss, validation_data=None): self.optimizer = optimizer self.layers = [] self.errors = {"training": [], "validation": []} self.loss_function = loss() self.X_val = np.empty([]) self.y_val = np.empty([]) if validation_data: self.X_val, self.y_val = validation_data self.y_val = to_categorical(self.y_val.astype("int"))
def gen_num_seq(nums): """ Method which generates sequence of numbers """ X = np.zeros([nums, 10, 20], dtype=float) y = np.zeros([nums, 10, 20], dtype=float) for i in range(nums): start = np.random.randint(0, 10) num_seq = np.arange(start, start + 10) X[i] = to_categorical(num_seq, n_col=20) y[i] = np.roll(X[i], -1, axis=0) y[:, -1, 1] = 1 # Mark endpoint as 1 return X, y
def gen_mult_ser(nums): """ Method which generates multiplication series """ X = np.zeros([nums, 10, 61], dtype=float) y = np.zeros([nums, 10, 61], dtype=float) for i in range(nums): start = np.random.randint(2, 7) mult_ser = np.linspace(start, start * 10, num=10, dtype=int) X[i] = to_categorical(mult_ser, n_col=61) y[i] = np.roll(X[i], -1, axis=0) y[:, -1, 1] = 1 # Mark endpoint as 1 return X, y
def fit(self, X, y): y = to_categorical(y) y_pred = np.zeros(np.shape(y)) for i in self.bar(range(self.n_estimators)): tree = self.trees[i] y_and_pred = np.concatenate((y, y_pred), axis=1) tree.fit(X, y_and_pred) update_pred = tree.predict(X) y_pred -= np.multiply(self.learning_rate, update_pred)
def __init__(self, optimizer, loss=CrossEntropy, validation_data=None): self.optimizer = optimizer self.layers = [] self.errors = {"training": [], "validation": []} self.loss_function = loss() self.validation_set = False if validation_data: self.validation_set = True self.X_val, self.y_val = validation_data # One-hot encoding of y self.y_val = to_categorical(self.y_val.astype("int"))
def fit(self, X, y): # Convert the categorical data to binary y = to_categorical(y.astype("int")) n_samples = np.shape(X)[0] n_batches = int(n_samples / self.batch_size) bar = progressbar.ProgressBar(widgets=bar_widgets) for _ in bar(range(self.n_iterations)): idx = range(n_samples) np.random.shuffle(idx) batch_t_error = 0 # Mean batch training error for i in range(n_batches): X_batch = X[idx[i * self.batch_size:(i + 1) * self.batch_size]] y_batch = y[idx[i * self.batch_size:(i + 1) * self.batch_size]] # Calculate output y_pred = self._forward_pass(X_batch) # Calculate the training loss loss = np.mean(self.loss_function.loss(y_batch, y_pred)) batch_t_error += loss loss_grad = self.loss_function.gradient(y_batch, y_pred) # Backprop. Update weights self._backward_pass(loss_grad=loss_grad) # Save the epoch mean error self.errors["training"].append(batch_t_error / n_batches) if self.X_val.any(): # Calculate the validation error y_val_p = self._forward_pass(self.X_val) validation_loss = np.mean( self.loss_function.loss(self.y_val, y_val_p)) self.errors["validation"].append(validation_loss)
def fit(self, X, y): y = to_categorical(y) super(GradientBoostingClassifier, self).fit(X, y)
# ........ # TRAIN # ........ print("Training:") print("- Adaboost") adaboost.fit(X_train, rescaled_y_train) print("- Decision Tree") decision_tree.fit(X_train, y_train) print("- Gradient Boosting") gbc.fit(X_train, y_train) print("- LDA") lda.fit(X_train, y_train) print("- Logistic Regression") logistic_regression.fit(X_train, y_train) print("- Multilayer Perceptron") mlp.fit(X_train, to_categorical(y_train), n_epochs=300, batch_size=50) print("- Naive Bayes") naive_bayes.fit(X_train, y_train) print("- Perceptron") perceptron.fit(X_train, to_categorical(y_train)) print("- Random Forest") random_forest.fit(X_train, y_train) print("- Support Vector Machine") support_vector_machine.fit(X_train, rescaled_y_train) print("- XGBoost") xgboost.fit(X_train, y_train) # ......... # PREDICT # ......... y_pred = {}
def main(): optimizer = Adam() #----- # MLP #----- data = datasets.load_digits() X = data.data y = data.target # Convert to one-hot encoding y = to_categorical(y.astype("int")) n_samples = np.shape(X) n_hidden = 512 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=1) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy, validation_data=(X_test, y_test)) clf.add(Dense(n_hidden, input_shape=(8 * 8, ))) clf.add(Activation('leaky_relu')) clf.add(Dense(n_hidden)) clf.add(Activation('leaky_relu')) clf.add(Dropout(0.25)) clf.add(Dense(n_hidden)) clf.add(Activation('leaky_relu')) clf.add(Dropout(0.25)) clf.add(Dense(n_hidden)) clf.add(Activation('leaky_relu')) clf.add(Dropout(0.25)) clf.add(Dense(10)) clf.add(Activation('softmax')) print() clf.summary(name="MLP") train_err, val_err = clf.fit(X_train, y_train, n_epochs=50, batch_size=256) # Training and validation error plot n = len(train_err) training, = plt.plot(range(n), train_err, label="Training Error") validation, = plt.plot(range(n), val_err, label="Validation Error") plt.legend(handles=[training, validation]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show() # Predict labels of the test data y_pred = np.argmax(clf.predict(X_test), axis=1) y_test = np.argmax(y_test, axis=1) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimension to 2D using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Multilayer Perceptron", accuracy=accuracy, legend_labels=range(10))
def main(): #---------- # Conv Net #---------- optimizer = Adam() data = datasets.load_digits() X = data.data y = data.target # Convert to one-hot encoding y = to_categorical(y.astype("int")) n_samples = np.shape(X) n_hidden = 512 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=1) # Reshape X to (n_samples, channels, height, width) X_train = X_train.reshape((-1, 1, 8, 8)) X_test = X_test.reshape((-1, 1, 8, 8)) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy, validation_data=(X_test, y_test)) clf.add( Conv2D(n_filters=16, filter_shape=(3, 3), input_shape=(1, 8, 8), padding='same')) clf.add(Activation('relu')) clf.add(Dropout(0.25)) clf.add(BatchNormalization()) clf.add(Conv2D(n_filters=32, filter_shape=(3, 3), padding='same')) clf.add(Activation('relu')) clf.add(Dropout(0.25)) clf.add(BatchNormalization()) clf.add(Flatten()) clf.add(Dense(256)) clf.add(Activation('relu')) clf.add(Dropout(0.5)) clf.add(BatchNormalization()) clf.add(Dense(10)) clf.add(Activation('softmax')) print() clf.summary(name="ConvNet") train_err, val_err = clf.fit(X_train, y_train, n_epochs=50, batch_size=256) # Training and validation error plot n = len(train_err) training, = plt.plot(range(n), train_err, label="Training Error") validation, = plt.plot(range(n), val_err, label="Validation Error") plt.legend(handles=[training, validation]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show() # Predict labels of the test data y_pred = np.argmax(clf.predict(X_test), axis=1) y_test = np.argmax(y_test, axis=1) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Flatten data set X_test = X_test.reshape(-1, 8 * 8) # Reduce dimension to 2D using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Convolutional Neural Network", accuracy=accuracy, legend_labels=range(10))