def main(): print("-- Classification Tree --") data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) clf = ClassificationTree() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) pca = PCA() pca.plot_in_2d(X_test, y_pred, title="Decision Tree", accuracy=accuracy, legend_labels=data.target_names) print("-- Regression Tree --") X, y = datasets.make_regression(n_features=1, n_samples=100, bias=0, noise=5) X_train, X_test, y_train, y_test = train_test_split(standardize(X), y, test_size=0.3) clf = RegressionTree() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) mse = mean_squared_error(y_test, y_pred) print("Mean Squared Error:", mse) # Plot the results plt.scatter(X_test[:, 0], y_test, color='black') plt.scatter(X_test[:, 0], y_pred, color='green') plt.title("Regression Tree (%.2f MSE)" % mse) plt.show()
def main(): print("-- Gradient Boosting Classification --") data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) clf = GradientBoostingClassifier(debug=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) Plot().plot_in_2d(X_test, y_pred, title="Gradient Boosting", accuracy=accuracy, legend_labels=data.target_names) print("-- Gradient Boosting Regression --") X, y = datasets.make_regression(n_features=1, n_samples=150, bias=0, noise=5) X_train, X_test, y_train, y_test = train_test_split(standardize(X), y, test_size=0.5) clf = GradientBoostingRegressor(debug=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) mse = mean_squared_error(y_test, y_pred) print("Mean Squared Error:", mse) # Plot the results plt.scatter(X_test[:, 0], y_test, color='black') plt.scatter(X_test[:, 0], y_pred, color='green') plt.title("Gradient Boosting Regression (%.2f MSE)" % mse) plt.show()
def main(): print("-- XGBoost --") data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=2) clf = XGBoost(debug=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) Plot().plot_in_2d(X_test, y_pred, title="XGBoost", accuracy=accuracy, legend_labels=data.target_names)
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target n_samples, n_features = np.shape(X) n_hidden, n_output = 50, 10 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=1) optimizer = GradientDescent_(learning_rate=0.0001, momentum=0.3) # MLP clf = MultilayerPerceptron(n_iterations=6000, batch_size=200, optimizer=optimizer, val_error=True) clf.add(DenseLayer(n_inputs=n_features, n_units=n_hidden)) clf.add(DenseLayer(n_inputs=n_hidden, n_units=n_hidden)) clf.add(DenseLayer(n_inputs=n_hidden, n_units=n_output, activation_function=Softmax)) clf.fit(X_train, y_train) clf.plot_errors() y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results pca = PCA() pca.plot_in_2d(X_test, y_pred, title="Multilayer Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) # Optimization method for finding weights that minimizes loss optimizer = RMSprop(learning_rate=0.01) # Perceptron clf = Perceptron(n_iterations=5000, activation_function=ELU, optimizer=optimizer, early_stopping=True, plot_errors=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def main(): # Load dataset data = datasets.load_iris() X = normalize(data.data[data.target != 0]) y = data.target[data.target != 0] y[y == 1] = 0 y[y == 2] = 1 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) clf = LogisticRegression(gradient_descent=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Logistic Regression", accuracy=accuracy)
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) # MLP clf = MultilayerPerceptron(n_hidden=12, n_iterations=1000, learning_rate=0.0001, momentum=0.8, activation_function=ExpLU, early_stopping=True, plot_errors=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results pca = PCA() pca.plot_in_2d(X_test, y_pred, title="Multilayer Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) # Optimization method for finding weights that minimizes loss optimizer = RMSprop(learning_rate=0.01) # Perceptron clf = Perceptron(n_iterations=5000, activation_function=ExpLU, optimizer=optimizer, early_stopping=True, plot_errors=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results pca = PCA() pca.plot_in_2d(X_test, y_pred, title="Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) # Perceptron clf = Perceptron(n_iterations=5000, learning_rate=0.001, activation_function=Sigmoid) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def main(): data = datasets.load_digits() X = data.data y = data.target digit1 = 1 digit2 = 8 idx = np.append(np.where(y == digit1)[0], np.where(y == digit2)[0]) y = data.target[idx] # Change labels to {-1, 1} y[y == digit1] = -1 y[y == digit2] = 1 X = data.data[idx] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5) # Adaboost classification with 5 weak classifiers clf = Adaboost(n_clf=5) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimensions to 2d using pca and plot the results Plot().plot_in_2d(X_test, y_pred, title="Adaboost", accuracy=accuracy)
def main(): print("-- Gradient Boosting Regression --") X, y = datasets.make_regression(n_features=1, n_samples=150, bias=0, noise=5) X_train, X_test, y_train, y_test = train_test_split(standardize(X), y, test_size=0.5) clf = GradientBoostingRegressor(debug=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) mse = mean_squared_error(y_test, y_pred) print("Mean Squared Error:", mse) # Plot the results plt.scatter(X_test[:, 0], y_test, color='black') plt.scatter(X_test[:, 0], y_pred, color='green') plt.title("Gradient Boosting Regression (%.2f MSE)" % mse) plt.show()
def main(): data = datasets.load_digits() X = data.data y = data.target digit1 = 1 digit2 = 8 idx = np.append(np.where(y == digit1)[0], np.where(y == digit2)[0]) y = data.target[idx] # Change labels to {-1, 1} y[y == digit1] = -1 y[y == digit2] = 1 X = data.data[idx] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5) # Adaboost classification with 5 weak classifiers clf = Adaboost(n_clf=5) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimensions to 2d using pca and plot the results Plot().plot_in_2d(X_test, y_pred, title="Adaboost", accuracy=accuracy)
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target # Convert the nominal y values to binary y = to_categorical(y) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) # MLP clf = MultilayerPerceptron(n_hidden=12, n_iterations=1000, learning_rate=0.01) clf.fit(X_train, y_train) y_pred = np.argmax(clf.predict(X_test), axis=1) y_test = np.argmax(y_test, axis=1) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Multilayer Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def main(): print ("-- Classification Tree --") data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) clf = ClassificationTree() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) Plot().plot_in_2d(X_test, y_pred, title="Decision Tree", accuracy=accuracy, legend_labels=data.target_names) print ("-- Regression Tree --") X, y = datasets.make_regression(n_features=1, n_samples=100, bias=0, noise=5) X_train, X_test, y_train, y_test = train_test_split(standardize(X), y, test_size=0.3) clf = RegressionTree() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) mse = mean_squared_error(y_test, y_pred) print ("Mean Squared Error:", mse) # Plot the results plt.scatter(X_test[:, 0], y_test, color='black') plt.scatter(X_test[:, 0], y_pred, color='green') plt.title("Regression Tree (%.2f MSE)" % mse) plt.show()
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33) clf = KNN(k=5) y_pred = clf.predict(X_test, X_train, y_train) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimensions to 2d using pca and plot the results Plot().plot_in_2d(X_test, y_pred, title="K Nearest Neighbors", accuracy=accuracy, legend_labels=data.target_names)
def fit(self, X, y): X_train = X y_train = y if self.val_error: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split( X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_batches = int(n_samples / self.batch_size) bar = progressbar.ProgressBar(widgets=bar_widgets) for i in bar(range(self.n_iterations)): X_, y_ = shuffle_data(X_train, y_train) batch_t_error = 0 # Mean batch training error batch_v_error = 0 # Mean batch validation error for idx in np.array_split(np.arange(n_samples), n_batches): X_batch, y_batch = X_[idx], y_[idx] # Calculate output y_pred = self._forward_pass(X_batch) # Calculate the cross entropy training loss loss = np.mean(self.cross_ent.loss(y_batch, y_pred)) batch_t_error += loss loss_grad = self.cross_ent.gradient(y_batch, y_pred) # Update the NN weights self._backward_pass(loss_grad=loss_grad) if self.val_error: # Calculate the validation error y_val_pred = self._forward_pass(X_validate) loss = np.mean(self.cross_ent.loss(y_validate, y_val_pred)) batch_v_error += loss batch_t_error /= n_batches batch_v_error /= n_batches self.errors["training"].append(batch_t_error) self.errors["validation"].append(batch_v_error)
def main(): data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=2) clf = RandomForest(debug=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) Plot().plot_in_2d(X_test, y_pred, title="Random Forest", accuracy=accuracy, legend_labels=data.target_names)
def main(): data = datasets.load_iris() X = normalize(data.data[data.target != 0]) y = data.target[data.target != 0] y[y == 1] = -1 y[y == 2] = 1 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33) clf = SupportVectorMachine(kernel=polynomial_kernel, power=4, coef=1) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Support Vector Machine", accuracy=accuracy)
def main(): data = datasets.load_iris() X = normalize(data.data) y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5) clf = NaiveBayes() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results pca = PCA() pca.plot_in_2d(X_test, y_pred, title="Naive Bayes", accuracy=accuracy, legend_labels=data.target_names)
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33) clf = KNN(k=5) y_pred = clf.predict(X_test, X_train, y_train) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimensions to 2d using pca and plot the results Plot().plot_in_2d(X_test, y_pred, title="K Nearest Neighbors", accuracy=accuracy, legend_labels=data.target_names)
def main(): data = datasets.load_digits() X = normalize(data.data) y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) # Perceptron clf = Perceptron(n_iterations=5000, learning_rate=0.001, activation_function=Sigmoid) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Perceptron", accuracy=accuracy, legend_labels=np.unique(y))
def main(): data = datasets.load_iris() X = normalize(data.data[data.target != 0]) y = data.target[data.target != 0] y[y == 1] = -1 y[y == 2] = 1 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33) clf = SupportVectorMachine(kernel=polynomial_kernel, power=4, coef=1) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results pca = PCA() pca.plot_in_2d(X_test, y_pred, title="Support Vector Machine", accuracy=accuracy)
def main(): # Load temperature data data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t") time = np.atleast_2d(data["time"].as_matrix()).T temp = np.atleast_2d(data["temp"].as_matrix()).T X = time # Time. Fraction of the year [0, 1] y = temp[:, 0] # Temperature. Reduce to one-dim X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) n_samples, n_features = np.shape(X) clf = LinearRegression() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) mse = mean_squared_error(y_test, y_pred) print("Mean squared error: %s" % (mse)) y_pred_line = clf.predict(X) # Color map cmap = plt.get_cmap('viridis') # Plot the results m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10) m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10) plt.plot(366 * X, y_pred_line, color='black', linewidth=2, label="Prediction") plt.suptitle("Polynomial Ridge Regression") plt.title("MSE: %.2f" % mse, fontsize=10) plt.xlabel('Day') plt.ylabel('Temperature in Celcius') plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right') plt.show()
def main(): # Load dataset data = datasets.load_iris() X = normalize(data.data[data.target != 0]) y = data.target[data.target != 0] y[y == 1] = 0 y[y == 2] = 1 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, seed=1) clf = LogisticRegression(gradient_descent=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Logistic Regression", accuracy=accuracy)
def main(): print("-- Regression Tree --") # Load temperature data data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t") time = np.atleast_2d(data["time"].as_matrix()).T temp = np.atleast_2d(data["temp"].as_matrix()).T X = standardize(time) # Time. Fraction of the year [0, 1] y = temp[:, 0] # Temperature. Reduce to one-dim X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3) clf = RegressionTree() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) y_pred_line = clf.predict(X) # Color map cmap = plt.get_cmap('viridis') mse = mean_squared_error(y_test, y_pred) print("Mean Squared Error:", mse) # Plot the results # Plot the results m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10) m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10) m3 = plt.scatter(366 * X_test, y_pred, color='black', s=10) plt.suptitle("Regression Tree") plt.title("MSE: %.2f" % mse, fontsize=10) plt.xlabel('Day') plt.ylabel('Temperature in Celcius') plt.legend((m1, m2, m3), ("Training data", "Test data", "Prediction"), loc='lower right') plt.show()
def main(): print ("-- Classification Tree --") data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) clf = ClassificationTree() clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) Plot().plot_in_2d(X_test, y_pred, title="Decision Tree", accuracy=accuracy, legend_labels=data.target_names)
def main(): # Load the dataset data = datasets.load_iris() X = data.data y = data.target # Three -> two classes X = X[y != 2] y = y[y != 2] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33) # Fit and predict using LDA lda = LDA() lda.fit(X_train, y_train) y_pred = lda.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) Plot().plot_in_2d(X_test, y_pred, title="LDA", accuracy=accuracy)
def main(): # Load the dataset data = datasets.load_iris() X = data.data y = data.target # Three -> two classes X = X[y != 2] y = y[y != 2] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33) # Fit and predict using LDA lda = LDA() lda.fit(X_train, y_train) y_pred = lda.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) Plot().plot_in_2d(X_test, y_pred, title="LDA", accuracy=accuracy)
def main(): print ("-- Gradient Boosting Classification --") data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) clf = GradientBoostingClassifier(debug=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) Plot().plot_in_2d(X_test, y_pred, title="Gradient Boosting", accuracy=accuracy, legend_labels=data.target_names)
def main(): data = datasets.load_iris() X = data.data y = data.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=2) clf = RandomForest(debug=True) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) pca = PCA() pca.plot_in_2d(X_test, y_pred, title="Random Forest", accuracy=accuracy, legend_labels=data.target_names)
def main(): data = datasets.load_digits() X = data.data y = data.target n_samples = np.shape(X) n_hidden = 512 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=1) optimizer = Adam() #----- # MLP #----- # clf = NeuralNetwork(optimizer=optimizer, # loss=CrossEntropy, # validation_data=(X_test, y_test)) # clf.add(Dense(n_hidden, input_shape=(8*8,))) # clf.add(Activation('leaky_relu')) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(10)) # clf.add(Activation('softmax')) # clf.fit(X_train, y_train, n_iterations=50, batch_size=256) # clf.plot_errors() # y_pred = np.argmax(clf.predict(X_test), axis=1) # accuracy = accuracy_score(y_test, y_pred) # print ("Accuracy:", accuracy) #---------- # Conv Net #---------- X_train = X_train.reshape((-1, 1, 8, 8)) X_test = X_test.reshape((-1, 1, 8, 8)) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy, validation_data=(X_test, y_test)) clf.add( Conv2D(n_filters=16, filter_shape=(3, 3), input_shape=(1, 8, 8), padding='same')) clf.add(Activation('relu')) clf.add(Conv2D(n_filters=32, filter_shape=(3, 3), padding='same')) clf.add(Activation('relu')) clf.add(Flatten()) clf.add(Dense(128)) clf.add(Activation('relu')) clf.add(Dense(10)) clf.add(Activation('softmax')) clf.fit(X_train, y_train, n_iterations=50, batch_size=256) clf.plot_errors() y_pred = np.argmax(clf.predict(X_test), axis=1) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) X_test = X_test.reshape(-1, 8 * 8) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Convolutional Neural Network", accuracy=accuracy, legend_labels=np.unique(y))
def main(): optimizer = Adam() def gen_mult_ser(nums): """ Method which generates multiplication series """ X = np.zeros([nums, 10, 61], dtype=float) y = np.zeros([nums, 10, 61], dtype=float) for i in range(nums): start = np.random.randint(2, 7) mult_ser = np.linspace(start, start * 10, num=10, dtype=int) X[i] = to_categorical(mult_ser, n_col=61) y[i] = np.roll(X[i], -1, axis=0) y[:, -1, 1] = 1 # Mark endpoint as 1 return X, y def gen_num_seq(nums): """ Method which generates sequence of numbers """ X = np.zeros([nums, 10, 20], dtype=float) y = np.zeros([nums, 10, 20], dtype=float) for i in range(nums): start = np.random.randint(0, 10) num_seq = np.arange(start, start + 10) X[i] = to_categorical(num_seq, n_col=20) y[i] = np.roll(X[i], -1, axis=0) y[:, -1, 1] = 1 # Mark endpoint as 1 return X, y X, y = gen_mult_ser(3000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) # Model definition clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy) clf.add(RNN(10, activation="tanh", bptt_trunc=5, input_shape=(10, 61))) clf.add(Activation('softmax')) clf.summary("RNN") # Print a problem instance and the correct solution tmp_X = np.argmax(X_train[0], axis=1) tmp_y = np.argmax(y_train[0], axis=1) print("Number Series Problem:") print("X = [" + " ".join(tmp_X.astype("str")) + "]") print("y = [" + " ".join(tmp_y.astype("str")) + "]") print() train_err, _ = clf.fit(X_train, y_train, n_epochs=500, batch_size=512) # Predict labels of the test data y_pred = np.argmax(clf.predict(X_test), axis=2) y_test = np.argmax(y_test, axis=2) print() print("Results:") for i in range(5): # Print a problem instance and the correct solution tmp_X = np.argmax(X_test[i], axis=1) tmp_y1 = y_test[i] tmp_y2 = y_pred[i] print("X = [" + " ".join(tmp_X.astype("str")) + "]") print("y_true = [" + " ".join(tmp_y1.astype("str")) + "]") print("y_pred = [" + " ".join(tmp_y2.astype("str")) + "]") print() accuracy = np.mean(accuracy_score(y_test, y_pred)) print("Accuracy:", accuracy) training = plt.plot(range(500), train_err, label="Training Error") plt.title("Error Plot") plt.ylabel('Training Error') plt.xlabel('Iterations') plt.show()
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, self.n_hidden)) + a self.w0 = (b - a) * np.random.random((1, self.n_hidden)) + a self.V = (b - a) * np.random.random((self.n_hidden, n_outputs)) + a self.v0 = (b - a) * np.random.random((1, n_outputs)) + a # Error history training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 for i in range(self.n_iterations): # Calculate hidden layer hidden_layer_input = X_train.dot(self.W) + self.w0 hidden_layer_output = self.activation.function(hidden_layer_input) # Calculate output layer output_layer_input = hidden_layer_output.dot(self.V) + self.v0 output = self.activation.function(output_layer_input) # Calculate the error error = y_train - output mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Calculate the loss gradient output_gradient = -2 * (y_train - output) * \ self.activation.gradient(output_layer_input) hidden_gradient = output_gradient.dot( self.V.T) * self.activation.gradient(hidden_layer_input) # Calcualte the gradient with respect to each weight term grad_wrt_v = hidden_layer_output.T.dot(output_gradient) grad_wrt_v0 = np.ones((1, n_samples)).dot(output_gradient) grad_wrt_w = X_train.T.dot(hidden_gradient) grad_wrt_w0 = np.ones((1, n_samples)).dot(hidden_gradient) # Update weights # Move against the gradient to minimize loss self.V = self.v_opt.update(w=self.V, grad_wrt_w=grad_wrt_v) self.v0 = self.v0_opt.update(w=self.v0, grad_wrt_w=grad_wrt_v0) self.W = self.w_opt.update(w=self.W, grad_wrt_w=grad_wrt_w) self.w0 = self.w0_opt.update(w=self.w0, grad_wrt_w=grad_wrt_w0) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # # If the validation error has been for more than 50 iterations # # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i+1), training_errors, label="Training Error") validation, = plt.plot(range(i+1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i+1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()
X = normalize(X) print("Dataset: The Digit Dataset (digits %s and %s)" % (digit1, digit2)) # .......................... # DIMENSIONALITY REDUCTION # .......................... pca = PCA() X = pca.transform(X, n_components=5) # Reduce to 5 dimensions n_samples, n_features = np.shape(X) # .......................... # TRAIN / TEST SPLIT # .......................... X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5) # Rescaled labels {-1, 1} rescaled_y_train = 2 * y_train - np.ones(np.shape(y_train)) rescaled_y_test = 2 * y_test - np.ones(np.shape(y_test)) # ....... # SETUP # ....... adaboost = Adaboost(n_clf=8) naive_bayes = NaiveBayes() knn = KNN(k=4) logistic_regression = LogisticRegression() mlp = NeuralNetwork(optimizer=Adam(), loss=CrossEntropy) mlp.add(Dense(input_shape=(n_features, ), n_units=64)) mlp.add(Activation('relu')) mlp.add(Dense(n_units=64))
X = normalize(X) print ("Dataset: The Digit Dataset (digits %s and %s)" % (digit1, digit2)) # .......................... # DIMENSIONALITY REDUCTION # .......................... pca = PCA() X = pca.transform(X, n_components=5) # Reduce to 5 dimensions n_samples, n_features = np.shape(X) # .......................... # TRAIN / TEST SPLIT # .......................... X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5) # Rescaled labels {-1, 1} rescaled_y_train = 2*y_train - np.ones(np.shape(y_train)) rescaled_y_test = 2*y_test - np.ones(np.shape(y_test)) # ....... # SETUP # ....... adaboost = Adaboost(n_clf = 8) naive_bayes = NaiveBayes() knn = KNN(k=4) logistic_regression = LogisticRegression() mlp = NeuralNetwork(optimizer=Adam(), loss=CrossEntropy) mlp.add(Dense(input_shape=(n_features,), n_units=64)) mlp.add(Activation('relu'))
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split( X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.w0 = (b - a) * np.random.random((1, n_outputs)) + a # Error history training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 # Lambda function that calculates the neuron outputs neuron_output = lambda w, b: self.activation.function( np.dot(X_train, w) + b) # Lambda function that calculates the loss gradient loss_grad = lambda w, b: -2 * (y_train - neuron_output(w, b)) * \ self.activation.gradient(np.dot(X_train, w) + b) # Lambda functions that calculates the gradient of the loss with # respect to each weight term. Allows for computation of loss gradient at different # coordinates. grad_func_wrt_w = lambda w: X_train.T.dot(loss_grad(w, self.w0)) grad_func_wrt_w0 = lambda b: np.ones( (1, n_samples)).dot(loss_grad(self.W, b)) # Optimize paramaters for n_iterations for i in range(self.n_iterations): # Training error error = y_train - neuron_output(self.W, self.w0) mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Update weights self.W = self.w_opt.update(w=self.W, grad_func=grad_func_wrt_w) self.w0 = self.w0_opt.update(w=self.w0, grad_func=grad_func_wrt_w0) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors ) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # If the validation error has been for more than 50 iterations # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i + 1), training_errors, label="Training Error") validation, = plt.plot(range(i + 1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i + 1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()
def main(): optimizer = Adam() #----- # MLP #----- data = datasets.load_digits() X = data.data y = data.target # Convert to one-hot encoding y = to_categorical(y.astype("int")) n_samples = np.shape(X) n_hidden = 512 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=1) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy, validation_data=(X_test, y_test)) clf.add(Dense(n_hidden, input_shape=(8 * 8, ))) clf.add(Activation('leaky_relu')) clf.add(Dense(n_hidden)) clf.add(Activation('leaky_relu')) clf.add(Dropout(0.25)) clf.add(Dense(n_hidden)) clf.add(Activation('leaky_relu')) clf.add(Dropout(0.25)) clf.add(Dense(n_hidden)) clf.add(Activation('leaky_relu')) clf.add(Dropout(0.25)) clf.add(Dense(10)) clf.add(Activation('softmax')) print() clf.summary(name="MLP") train_err, val_err = clf.fit(X_train, y_train, n_epochs=50, batch_size=256) # Training and validation error plot n = len(train_err) training, = plt.plot(range(n), train_err, label="Training Error") validation, = plt.plot(range(n), val_err, label="Validation Error") plt.legend(handles=[training, validation]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show() # Predict labels of the test data y_pred = np.argmax(clf.predict(X_test), axis=1) y_test = np.argmax(y_test, axis=1) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) # Reduce dimension to 2D using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Multilayer Perceptron", accuracy=accuracy, legend_labels=range(10))
def main(): data = datasets.load_digits() X = data.data y = data.target n_samples = np.shape(X) n_hidden = 512 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=1) optimizer = Adam() #----- # MLP #----- # clf = NeuralNetwork(optimizer=optimizer, # loss=CrossEntropy, # validation_data=(X_test, y_test)) # clf.add(Dense(n_hidden, input_shape=(8*8,))) # clf.add(Activation('leaky_relu')) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(10)) # clf.add(Activation('softmax')) # print () # clf.summary(name="MLP") # clf.fit(X_train, y_train, n_epochs=50, batch_size=256) # clf.plot_errors() # y_pred = np.argmax(clf.predict(X_test), axis=1) # accuracy = accuracy_score(y_test, y_pred) # print ("Accuracy:", accuracy) #---------- # Conv Net #---------- X_train = X_train.reshape((-1, 1, 8, 8)) X_test = X_test.reshape((-1, 1, 8, 8)) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy, validation_data=(X_test, y_test)) clf.add( Conv2D(n_filters=16, filter_shape=(3, 3), input_shape=(1, 8, 8), padding='same')) clf.add(Activation('relu')) clf.add(Dropout(0.25)) clf.add(BatchNormalization()) clf.add(Conv2D(n_filters=32, filter_shape=(3, 3), padding='same')) clf.add(Activation('relu')) clf.add(Dropout(0.25)) clf.add(BatchNormalization()) clf.add(Flatten()) clf.add(Dense(256)) clf.add(Activation('relu')) clf.add(Dropout(0.5)) clf.add(BatchNormalization()) clf.add(Dense(10)) clf.add(Activation('softmax')) print() clf.summary(name="ConvNet") train_err, val_err = clf.fit(X_train, y_train, n_epochs=50, batch_size=256) # Training and validation error plot n = len(train_err) training, = plt.plot(range(n), train_err, label="Training Error") validation, = plt.plot(range(n), val_err, label="Validation Error") plt.legend(handles=[training, validation]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show() # Make a prediction of the test set y_pred = np.argmax(clf.predict(X_test), axis=1) accuracy = accuracy_score(y_test, y_pred) print("Accuracy:", accuracy) X_test = X_test.reshape(-1, 8 * 8) # Reduce dimension to 2D using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Convolutional Neural Network", accuracy=accuracy, legend_labels=np.unique(y))
def main(): # Load temperature data data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t") time = np.atleast_2d(data["time"].values).T temp = np.atleast_2d(data["temp"].values).T X = time # fraction of the year [0, 1] y = temp X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) n_samples, n_features = np.shape(X) # Prior parameters # - Weights are assumed distr. according to a Normal distribution # - The variance of the weights are assumed distributed according to # a scaled inverse chi-squared distribution. # High prior uncertainty! # Normal mu0 = np.array([0] * n_features) omega0 = np.diag([.0001] * n_features) # Scaled inverse chi-squared nu0 = 1 sigma_sq0 = 100 # The credible interval cred_int = 10 clf = BayesianRegression(n_draws=2000, poly_degree=4, mu0=mu0, omega0=omega0, nu0=nu0, sigma_sq0=sigma_sq0, cred_int=cred_int) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) mse = mean_squared_error(y_test, y_pred) # Get prediction line y_pred_, y_lower_, y_upper_ = clf.predict(X=X, eti=True) # Print the mean squared error print ("Mean Squared Error:", mse) # Color map cmap = plt.get_cmap('viridis') # Plot the results m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10) m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10) p1 = plt.plot(366 * X, y_pred_, color="black", linewidth=2, label="Prediction") p2 = plt.plot(366 * X, y_lower_, color="gray", linewidth=2, label="{0}% Credible Interval".format(cred_int)) p3 = plt.plot(366 * X, y_upper_, color="gray", linewidth=2) plt.axis((0, 366, -20, 25)) plt.suptitle("Bayesian Regression") plt.title("MSE: %.2f" % mse, fontsize=10) plt.xlabel('Day') plt.ylabel('Temperature in Celcius') plt.legend(loc='lower right') # plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right') plt.legend(loc='lower right') plt.show()
def main(): data = datasets.load_digits() X = data.data y = data.target n_samples = np.shape(X) n_hidden = 512 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, seed=1) optimizer = Adam() #----- # MLP #----- # clf = NeuralNetwork(optimizer=optimizer, # loss=CrossEntropy, # validation_data=(X_test, y_test)) # clf.add(Dense(n_hidden, input_shape=(8*8,))) # clf.add(Activation('leaky_relu')) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(n_hidden)) # clf.add(Activation('leaky_relu')) # clf.add(Dropout(0.25)) # clf.add(Dense(10)) # clf.add(Activation('softmax')) # clf.fit(X_train, y_train, n_iterations=50, batch_size=256) # clf.plot_errors() # y_pred = np.argmax(clf.predict(X_test), axis=1) # accuracy = accuracy_score(y_test, y_pred) # print ("Accuracy:", accuracy) #---------- # Conv Net #---------- X_train = X_train.reshape((-1,1,8,8)) X_test = X_test.reshape((-1,1,8,8)) clf = NeuralNetwork(optimizer=optimizer, loss=CrossEntropy, validation_data=(X_test, y_test)) clf.add(Conv2D(n_filters=16, filter_shape=(3,3), input_shape=(1,8,8), padding='same')) clf.add(Activation('relu')) clf.add(Conv2D(n_filters=32, filter_shape=(3,3), padding='same')) clf.add(Activation('relu')) clf.add(Flatten()) clf.add(Dense(128)) clf.add(Activation('relu')) clf.add(Dense(10)) clf.add(Activation('softmax')) clf.fit(X_train, y_train, n_iterations=50, batch_size=256) clf.plot_errors() y_pred = np.argmax(clf.predict(X_test), axis=1) accuracy = accuracy_score(y_test, y_pred) print ("Accuracy:", accuracy) X_test = X_test.reshape(-1, 8*8) # Reduce dimension to two using PCA and plot the results Plot().plot_in_2d(X_test, y_pred, title="Convolutional Neural Network", accuracy=accuracy, legend_labels=np.unique(y))
def main(): # Load temperature data data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t") time = np.atleast_2d(data["time"].as_matrix()).T temp = np.atleast_2d(data["temp"].as_matrix()).T X = time # fraction of the year [0, 1] y = temp X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) n_samples, n_features = np.shape(X) # Prior parameters # - Weights are assumed distr. according to a Normal distribution # - The variance of the weights are assumed distributed according to # a scaled inverse chi-squared distribution. # High prior uncertainty! # Normal mu0 = np.array([0] * n_features) omega0 = np.diag([.0001] * n_features) # Scaled inverse chi-squared nu0 = 1 sigma_sq0 = 100 # The credible interval cred_int = 10 clf = BayesianRegression(n_draws=2000, poly_degree=4, mu0=mu0, omega0=omega0, nu0=nu0, sigma_sq0=sigma_sq0, cred_int=cred_int) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) mse = mean_squared_error(y_test, y_pred) # Get prediction line y_pred_, y_lower_, y_upper_ = clf.predict(X=X, eti=True) # Print the mean squared error print("Mean Squared Error:", mse) # Color map cmap = plt.get_cmap('viridis') # Plot the results m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10) m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10) p1 = plt.plot(366 * X, y_pred_, color="black", linewidth=2, label="Prediction") p2 = plt.plot(366 * X, y_lower_, color="gray", linewidth=2, label="{0}% Credible Interval".format(cred_int)) p3 = plt.plot(366 * X, y_upper_, color="gray", linewidth=2) plt.axis((0, 366, -20, 25)) plt.suptitle("Bayesian Regression") plt.title("MSE: %.2f" % mse, fontsize=10) plt.xlabel('Day') plt.ylabel('Temperature in Celcius') plt.legend(loc='lower right') # plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right') plt.legend(loc='lower right') plt.show()
def main(): # Load temperature data data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t") time = np.atleast_2d(data["time"].as_matrix()).T temp = data["temp"].as_matrix() X = time # fraction of the year [0, 1] y = temp X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4) poly_degree = 11 # Finding regularization constant using cross validation lowest_error = float("inf") best_reg_factor = None print ("Finding regularization constant using cross validation:") k = 10 for reg_factor in np.arange(0, 0.1, 0.01): cross_validation_sets = k_fold_cross_validation_sets( X_train, y_train, k=k) mse = 0 for _X_train, _X_test, _y_train, _y_test in cross_validation_sets: clf = PolynomialRidgeRegression(degree=poly_degree, reg_factor=reg_factor, learning_rate=0.001, n_iterations=10000) clf.fit(_X_train, _y_train) y_pred = clf.predict(_X_test) _mse = mean_squared_error(_y_test, y_pred) mse += _mse mse /= k # Print the mean squared error print ("\tMean Squared Error: %s (regularization: %s)" % (mse, reg_factor)) # Save reg. constant that gave lowest error if mse < lowest_error: best_reg_factor = reg_factor lowest_error = mse # Make final prediction clf = PolynomialRidgeRegression(degree=poly_degree, reg_factor=best_reg_factor, learning_rate=0.001, n_iterations=10000) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) mse = mean_squared_error(y_test, y_pred) print ("Mean squared error: %s (given by reg. factor: %s)" % (lowest_error, best_reg_factor)) y_pred_line = clf.predict(X) # Color map cmap = plt.get_cmap('viridis') # Plot the results m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10) m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10) plt.plot(366 * X, y_pred_line, color='black', linewidth=2, label="Prediction") plt.suptitle("Polynomial Ridge Regression") plt.title("MSE: %.2f" % mse, fontsize=10) plt.xlabel('Day') plt.ylabel('Temperature in Celcius') plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right') plt.show()
def fit(self, X, y): X_train = X y_train = y if self.early_stopping: # Split the data into training and validation sets X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.1) y_validate = categorical_to_binary(y_validate) # Convert the nominal y values to binary y_train = categorical_to_binary(y_train) n_samples, n_features = np.shape(X_train) n_outputs = np.shape(y_train)[1] # Initial weights between [-1/sqrt(N), 1/sqrt(N)] a = -1 / math.sqrt(n_features) b = -a self.W = (b - a) * np.random.random((n_features, n_outputs)) + a self.w0 = (b - a) * np.random.random((1, n_outputs)) + a # Error history training_errors = [] validation_errors = [] iter_with_rising_val_error = 0 # Lambda function that calculates the neuron outputs neuron_output = lambda w, b: self.activation.function(np.dot(X_train, w) + b) # Lambda function that calculates the loss gradient loss_grad = lambda w, b: -2 * (y_train - neuron_output(w, b)) * \ self.activation.gradient(np.dot(X_train, w) + b) # Lambda functions that calculates the gradient of the loss with # respect to each weight term. Allows for computation of loss gradient at different # coordinates. grad_func_wrt_w = lambda w: X_train.T.dot(loss_grad(w, self.w0)) grad_func_wrt_w0 = lambda b: np.ones((1, n_samples)).dot(loss_grad(self.W, b)) # Optimize paramaters for n_iterations for i in range(self.n_iterations): # Training error error = y_train - neuron_output(self.W, self.w0) mse = np.mean(np.power(error, 2)) training_errors.append(mse) # Update weights self.W = self.w_opt.update(w=self.W, grad_func=grad_func_wrt_w) self.w0 = self.w0_opt.update(w=self.w0, grad_func=grad_func_wrt_w0) if self.early_stopping: # Calculate the validation error error = y_validate - self._calculate_output(X_validate) mse = np.mean(np.power(error, 2)) validation_errors.append(mse) # If the validation error is larger than the previous iteration increase # the counter if len(validation_errors) > 1 and validation_errors[-1] > validation_errors[-2]: iter_with_rising_val_error += 1 # If the validation error has been for more than 50 iterations # stop training to avoid overfitting if iter_with_rising_val_error > 50: break else: iter_with_rising_val_error = 0 # Plot the training error if self.plot_errors: if self.early_stopping: # Training and validation error plot training, = plt.plot(range(i+1), training_errors, label="Training Error") validation, = plt.plot(range(i+1), validation_errors, label="Validation Error") plt.legend(handles=[training, validation]) else: training, = plt.plot(range(i+1), training_errors, label="Training Error") plt.legend(handles=[training]) plt.title("Error Plot") plt.ylabel('Error') plt.xlabel('Iterations') plt.show()