def main():
data = datasets.load_digits()
X = normalize(data.data)
y = data.target
# Convert the nominal y values to binary
y = to_categorical(y)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
seed=1)
# MLP
clf = MultilayerPerceptron(n_hidden=12,
n_iterations=1000,
learning_rate=0.01)
clf.fit(X_train, y_train)
y_pred = np.argmax(clf.predict(X_test), axis=1)
y_test = np.argmax(y_test, axis=1)
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)
# Reduce dimension to two using PCA and plot the results
Plot().plot_in_2d(X_test,
y_pred,
title="Multilayer Perceptron",
accuracy=accuracy,
legend_labels=np.unique(y))
def fit(self, X, y):
X_train = X
y_train = y
# One-hot encoding of nominal y-values
y_train = to_categorical(y_train)
n_samples, n_features = np.shape(X_train)
n_outputs = np.shape(y_train)[1]
# Initialize weights between [-1/sqrt(N), 1/sqrt(N)]
limit = -1 / math.sqrt(n_features)
self.W = np.random.uniform(-limit, limit, (n_features, n_outputs))
self.w0 = np.zeros((1, n_outputs))
for i in range(self.n_iterations):
# Calculate outputs
linear_output = np.dot(X_train, self.W) + self.w0
y_pred = self.activation.function(linear_output)
# Calculate the loss gradient
error_gradient = -2 * (y_train - y_pred) * \
self.activation.gradient(linear_output)
# Calculate the gradient of the loss with respect to each weight term
grad_wrt_w = X_train.T.dot(error_gradient)
grad_wrt_w0 = np.ones((1, n_samples)).dot(error_gradient)
# Update weights
self.W -= self.learning_rate * grad_wrt_w
self.w0 -= self.learning_rate * grad_wrt_w0
def main():
data = datasets.load_digits()
X = normalize(data.data)
y = data.target
# One-hot encoding of nominal y-values
y = to_categorical(y)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
seed=1)
# Perceptron
clf = Perceptron(n_iterations=5000,
learning_rate=0.001,
activation_function=Sigmoid)
clf.fit(X_train, y_train)
y_pred = np.argmax(clf.predict(X_test), axis=1)
y_test = np.argmax(y_test, axis=1)
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)
# Reduce dimension to two using PCA and plot the results
Plot().plot_in_2d(X_test,
y_pred,
title="Perceptron",
accuracy=accuracy,
legend_labels=np.unique(y))
def fit(self, X, y, n_epochs, batch_size):
# Convert to one-hot encoding
y = to_categorical(y.astype("int"))
n_samples = np.shape(X)[0]
n_batches = int(n_samples / batch_size)
bar = progressbar.ProgressBar(widgets=bar_widgets)
for _ in bar(range(n_epochs)):
idx = range(n_samples)
np.random.shuffle(idx)
batch_t_error = 0 # Mean batch training error
for i in range(n_batches):
X_batch = X[idx[i * batch_size:(i + 1) * batch_size]]
y_batch = y[idx[i * batch_size:(i + 1) * batch_size]]
loss, _ = self.train_on_batch(X_batch, y_batch)
batch_t_error += loss
# Save the epoch mean error
self.errors["training"].append(batch_t_error / n_batches)
if self.X_val.any():
# Calculate the validation error
y_val_p = self._forward_pass(self.X_val)
validation_loss = np.mean(
self.loss_function.loss(self.y_val, y_val_p))
self.errors["validation"].append(validation_loss)
return self.errors["training"], self.errors["validation"]
def fit(self, X, y):
X_train = X
y_train = y
# One-hot encoding of nominal y-values
y_train = to_categorical(y_train)
n_samples, n_features = np.shape(X_train)
n_outputs = np.shape(y_train)[1]
# Initialize weights between [-1/sqrt(N), 1/sqrt(N)]
limit = 1 / math.sqrt(n_features)
self.W = np.random.uniform(-limit, limit, (n_features, n_outputs))
self.w0 = np.zeros((1, n_outputs))
for i in range(self.n_iterations):
# Calculate outputs
linear_output = np.dot(X_train, self.W) + self.w0
y_pred = self.activation.function(linear_output)
# Calculate the loss gradient
error_gradient = -2 * (y_train - y_pred) * \
self.activation.gradient(linear_output)
# Calculate the gradient of the loss with respect to each weight term
grad_wrt_w = X_train.T.dot(error_gradient)
grad_wrt_w0 = np.ones((1, n_samples)).dot(error_gradient)
# Update weights
self.W -= self.learning_rate * grad_wrt_w
self.w0 -= self.learning_rate * grad_wrt_w0
def fit(self, X, y, n_iterations, batch_size):
# Convert to one-hot encoding
y = to_categorical(y.astype("int"))
n_samples = np.shape(X)[0]
n_batches = int(n_samples / batch_size)
bar = progressbar.ProgressBar(widgets=bar_widgets)
for _ in bar(range(n_iterations)):
idx = range(n_samples)
np.random.shuffle(idx)
batch_t_error = 0 # Mean batch training error
for i in range(n_batches):
X_batch = X[idx[i*batch_size:(i+1)*batch_size]]
y_batch = y[idx[i*batch_size:(i+1)*batch_size]]
loss, _ = self.train_on_batch(X_batch, y_batch)
batch_t_error += loss
# Save the epoch mean error
self.errors["training"].append(batch_t_error / n_batches)
if self.X_val.any():
# Calculate the validation error
y_val_p = self._forward_pass(self.X_val)
validation_loss = np.mean(self.loss_function.loss(self.y_val, y_val_p))
self.errors["validation"].append(validation_loss)
def __init__(self, optimizer, loss, validation_data=None):
self.optimizer = optimizer
self.layers = []
self.errors = {"training": [], "validation": []}
self.loss_function = loss()
self.X_val = np.empty([])
self.y_val = np.empty([])
if validation_data:
self.X_val, self.y_val = validation_data
self.y_val = to_categorical(self.y_val.astype("int"))
def __init__(self, optimizer, loss, validation_data=None):
self.optimizer = optimizer
self.layers = []
self.errors = {"training": [], "validation": []}
self.loss_function = loss()
self.X_val = np.empty([])
self.y_val = np.empty([])
if validation_data:
self.X_val, self.y_val = validation_data
self.y_val = to_categorical(self.y_val.astype("int"))
def gen_num_seq(nums):
""" Method which generates sequence of numbers """
X = np.zeros([nums, 10, 20], dtype=float)
y = np.zeros([nums, 10, 20], dtype=float)
for i in range(nums):
start = np.random.randint(0, 10)
num_seq = np.arange(start, start + 10)
X[i] = to_categorical(num_seq, n_col=20)
y[i] = np.roll(X[i], -1, axis=0)
y[:, -1, 1] = 1 # Mark endpoint as 1
return X, y
def gen_mult_ser(nums):
""" Method which generates multiplication series """
X = np.zeros([nums, 10, 61], dtype=float)
y = np.zeros([nums, 10, 61], dtype=float)
for i in range(nums):
start = np.random.randint(2, 7)
mult_ser = np.linspace(start, start * 10, num=10, dtype=int)
X[i] = to_categorical(mult_ser, n_col=61)
y[i] = np.roll(X[i], -1, axis=0)
y[:, -1, 1] = 1 # Mark endpoint as 1
return X, y
def fit(self, X, y):
y = to_categorical(y)
y_pred = np.zeros(np.shape(y))
for i in self.bar(range(self.n_estimators)):
tree = self.trees[i]
y_and_pred = np.concatenate((y, y_pred), axis=1)
tree.fit(X, y_and_pred)
update_pred = tree.predict(X)
y_pred -= np.multiply(self.learning_rate, update_pred)
def __init__(self, optimizer, loss=CrossEntropy, validation_data=None):
self.optimizer = optimizer
self.layers = []
self.errors = {"training": [], "validation": []}
self.loss_function = loss()
self.validation_set = False
if validation_data:
self.validation_set = True
self.X_val, self.y_val = validation_data
# One-hot encoding of y
self.y_val = to_categorical(self.y_val.astype("int"))
def fit(self, X, y):
# Convert the categorical data to binary
y = to_categorical(y.astype("int"))
n_samples = np.shape(X)[0]
n_batches = int(n_samples / self.batch_size)
bar = progressbar.ProgressBar(widgets=bar_widgets)
for _ in bar(range(self.n_iterations)):
idx = range(n_samples)
np.random.shuffle(idx)
batch_t_error = 0 # Mean batch training error
for i in range(n_batches):
X_batch = X[idx[i * self.batch_size:(i + 1) * self.batch_size]]
y_batch = y[idx[i * self.batch_size:(i + 1) * self.batch_size]]
# Calculate output
y_pred = self._forward_pass(X_batch)
# Calculate the training loss
loss = np.mean(self.loss_function.loss(y_batch, y_pred))
batch_t_error += loss
loss_grad = self.loss_function.gradient(y_batch, y_pred)
# Backprop. Update weights
self._backward_pass(loss_grad=loss_grad)
# Save the epoch mean error
self.errors["training"].append(batch_t_error / n_batches)
if self.X_val.any():
# Calculate the validation error
y_val_p = self._forward_pass(self.X_val)
validation_loss = np.mean(
self.loss_function.loss(self.y_val, y_val_p))
self.errors["validation"].append(validation_loss)
def fit(self, X, y):
y = to_categorical(y)
super(GradientBoostingClassifier, self).fit(X, y)
# ........
# TRAIN
# ........
print("Training:")
print("- Adaboost")
adaboost.fit(X_train, rescaled_y_train)
print("- Decision Tree")
decision_tree.fit(X_train, y_train)
print("- Gradient Boosting")
gbc.fit(X_train, y_train)
print("- LDA")
lda.fit(X_train, y_train)
print("- Logistic Regression")
logistic_regression.fit(X_train, y_train)
print("- Multilayer Perceptron")
mlp.fit(X_train, to_categorical(y_train), n_epochs=300, batch_size=50)
print("- Naive Bayes")
naive_bayes.fit(X_train, y_train)
print("- Perceptron")
perceptron.fit(X_train, to_categorical(y_train))
print("- Random Forest")
random_forest.fit(X_train, y_train)
print("- Support Vector Machine")
support_vector_machine.fit(X_train, rescaled_y_train)
print("- XGBoost")
xgboost.fit(X_train, y_train)
# .........
# PREDICT
# .........
y_pred = {}
def main():
optimizer = Adam()
#-----
# MLP
#-----
data = datasets.load_digits()
X = data.data
y = data.target
# Convert to one-hot encoding
y = to_categorical(y.astype("int"))
n_samples = np.shape(X)
n_hidden = 512
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.4,
seed=1)
clf = NeuralNetwork(optimizer=optimizer,
loss=CrossEntropy,
validation_data=(X_test, y_test))
clf.add(Dense(n_hidden, input_shape=(8 * 8, )))
clf.add(Activation('leaky_relu'))
clf.add(Dense(n_hidden))
clf.add(Activation('leaky_relu'))
clf.add(Dropout(0.25))
clf.add(Dense(n_hidden))
clf.add(Activation('leaky_relu'))
clf.add(Dropout(0.25))
clf.add(Dense(n_hidden))
clf.add(Activation('leaky_relu'))
clf.add(Dropout(0.25))
clf.add(Dense(10))
clf.add(Activation('softmax'))
print()
clf.summary(name="MLP")
train_err, val_err = clf.fit(X_train, y_train, n_epochs=50, batch_size=256)
# Training and validation error plot
n = len(train_err)
training, = plt.plot(range(n), train_err, label="Training Error")
validation, = plt.plot(range(n), val_err, label="Validation Error")
plt.legend(handles=[training, validation])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
# Predict labels of the test data
y_pred = np.argmax(clf.predict(X_test), axis=1)
y_test = np.argmax(y_test, axis=1)
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)
# Reduce dimension to 2D using PCA and plot the results
Plot().plot_in_2d(X_test,
y_pred,
title="Multilayer Perceptron",
accuracy=accuracy,
legend_labels=range(10))
def main():
#----------
# Conv Net
#----------
optimizer = Adam()
data = datasets.load_digits()
X = data.data
y = data.target
# Convert to one-hot encoding
y = to_categorical(y.astype("int"))
n_samples = np.shape(X)
n_hidden = 512
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.4,
seed=1)
# Reshape X to (n_samples, channels, height, width)
X_train = X_train.reshape((-1, 1, 8, 8))
X_test = X_test.reshape((-1, 1, 8, 8))
clf = NeuralNetwork(optimizer=optimizer,
loss=CrossEntropy,
validation_data=(X_test, y_test))
clf.add(
Conv2D(n_filters=16,
filter_shape=(3, 3),
input_shape=(1, 8, 8),
padding='same'))
clf.add(Activation('relu'))
clf.add(Dropout(0.25))
clf.add(BatchNormalization())
clf.add(Conv2D(n_filters=32, filter_shape=(3, 3), padding='same'))
clf.add(Activation('relu'))
clf.add(Dropout(0.25))
clf.add(BatchNormalization())
clf.add(Flatten())
clf.add(Dense(256))
clf.add(Activation('relu'))
clf.add(Dropout(0.5))
clf.add(BatchNormalization())
clf.add(Dense(10))
clf.add(Activation('softmax'))
print()
clf.summary(name="ConvNet")
train_err, val_err = clf.fit(X_train, y_train, n_epochs=50, batch_size=256)
# Training and validation error plot
n = len(train_err)
training, = plt.plot(range(n), train_err, label="Training Error")
validation, = plt.plot(range(n), val_err, label="Validation Error")
plt.legend(handles=[training, validation])
plt.title("Error Plot")
plt.ylabel('Error')
plt.xlabel('Iterations')
plt.show()
# Predict labels of the test data
y_pred = np.argmax(clf.predict(X_test), axis=1)
y_test = np.argmax(y_test, axis=1)
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)
# Flatten data set
X_test = X_test.reshape(-1, 8 * 8)
# Reduce dimension to 2D using PCA and plot the results
Plot().plot_in_2d(X_test,
y_pred,
title="Convolutional Neural Network",
accuracy=accuracy,
legend_labels=range(10))
