def __init__(self,
train,
valid,
test,
learningRate=0.01,
epochs=50,
activation='sigmoid',
error='mse'):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
# Initialize the weight vector with small values
self.weight = 0.01 * np.random.randn(1,
self.trainingSet.input.shape[1])
weight_Plus_bias = np.insert(self.weight, 0, 1, axis=1)
self.weight = weight_Plus_bias
# Choose the error function
self.errorString = error
self._initialize_error(error)
#initialize also the layer
self.layer = LogisticLayer(nIn=self.trainingSet.input.shape[1],
nOut=1,
activation='sigmoid',
weights=weight_Plus_bias)
def __init__(self, train, valid, test, learning_rate=0.01, epochs=50):
self.learning_rate = learning_rate
self.epochs = epochs
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
# Use a logistic layer as one-neuron classification (output) layer
self.layer = LogisticLayer(train.input.shape[1],
1,
is_classifier_layer=True)
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input,
0,
1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input,
0,
1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def __init__(self, data, learningRate=0.01, epochs=50, hiddensize=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = data.trainingSet
self.validationSet = data.validationSet
self.testSet = data.testSet
self.data=data
self.layer=LogisticLayer(data.trainingSet.input.shape[1],hiddensize,learningRate)
# Initialize the weight vector with small values
self.weight = 0.01*np.random.randn(self.layer.size)
def __init__(self, train, valid, test, learningRate=0.01, epochs=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
self.logisticLayer = LogisticLayer(len(self.trainingSet.input[0]),
1)
def __init__(self,
train,
valid,
test,
learningRate=0.01,
epochs=50,
loss='bce'):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
if loss == 'bce':
self.loss = BinaryCrossEntropyError()
elif loss == 'sse':
self.loss = SumSquaredError()
elif loss == 'mse':
self.loss = MeanSquaredError()
elif loss == 'different':
self.loss = DifferentError()
elif loss == 'absolute':
self.loss = AbsoluteError()
else:
raise ValueError('There is no predefined loss function ' +
'named ' + str)
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
# Use a logistic layer as one-neuron classification (output) layer
self.layer = LogisticLayer(train.input.shape[1],
1,
activation='sigmoid',
isClassifierLayer=True)
# add bias values ("1"s) at the beginning of all data sets
self.trainingSet.input = np.insert(self.trainingSet.input,
0,
1,
axis=1)
self.validationSet.input = np.insert(self.validationSet.input,
0,
1,
axis=1)
self.testSet.input = np.insert(self.testSet.input, 0, 1, axis=1)
def _costructNetwork(self, netStruct, activationFunctions):
prevSize = self.trainingSet.input.shape[1] - 1
for (size, func) in zip(netStruct, activationFunctions):
self.layers.append(LogisticLayer(prevSize, size, None, func, False))
prevSize = size
self.layers[-1].isClassifierLayer = True
def __init__(self, train, valid, test, learningRate=0.01, epochs=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
self.layer = LogisticLayer(self.trainingSet.input.shape[1], 1, learning_rate=learningRate)
def main(args):
hidden_layers = [
LogisticLayer(128, 128, isClassifierLayer=True)
for layer in range(args.num_layers)
]
data = MNISTSeven(args.dataset, 3000, 1000, 1000, oneHot=True)
MLP = MultilayerPerceptron(data.trainingSet, data.validationSet,
data.testSet, hidden_layers)
MLP.train(verbose=True)
def __init__(self, train, valid, test, learning_rate=0.01, epochs=50):
self.learning_rate = learning_rate
self.epochs = epochs
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
# Use a logistic layer as one-neuron classification (output) layer
self.layer = LogisticLayer(train.input.shape[1], 1,
is_classifier_layer=True)
def main():
#data = MNISTSeven("../data/mnist_seven.csv", 3000, 1000, 1000,
# one_hot=True, target_digit='7')
# NOTE:
# Comment out the MNISTSeven instantiation above and
# uncomment the following to work with full MNIST task
data = MNISTSeven("../data/mnist_seven.csv",
3000,
1000,
1000,
one_hot=False)
# # NOTE:
# # Other 1-digit classifiers do not make sense now for comparison purpose
# # So you should comment them out, let alone the MLP training and evaluation
#
# # Train the classifiers #
# print("=========================")
# print("Training..")
#
# # Stupid Classifier
# myStupidClassifier = StupidRecognizer(data.training_set,
# data.validation_set,
# data.test_set)
#
# print("\nStupid Classifier has been training..")
# myStupidClassifier.train()
# print("Done..")
# # Do the recognizer
# # Explicitly specify the test set to be evaluated
# stupidPred = myStupidClassifier.evaluate()
#
# # Perceptron
# myPerceptronClassifier = Perceptron(data.training_set,
# data.validation_set,
# data.test_set,
# learning_rate=0.005,
# epochs=10)
#
# print("\nPerceptron has been training..")
# myPerceptronClassifier.train()
# print("Done..")
# # Do the recognizer
# # Explicitly specify the test set to be evaluated
# perceptronPred = myPerceptronClassifier.evaluate()
#
# # Logistic Regression
# myLRClassifier = LogisticRegression(data.training_set,
# data.validation_set,
# data.test_set,
# learning_rate=0.005,
# epochs=30)
#
# print("\nLogistic Regression has been training..")
# myLRClassifier.train()
# print("Done..")
# # Do the recognizer
# # Explicitly specify the test set to be evaluated
# lrPred = myLRClassifier.evaluate()
# Build up the network from specific layers
# Here is an example of a MLP acting like the Logistic Regression
layers = []
layers.append(LogisticLayer(784, 5, None, "sigmoid", True))
layers.append(LogisticLayer(5, 10, None, "softmax", False))
myMLPClassifier = MultilayerPerceptron(data.training_set,
data.validation_set,
data.test_set,
learning_rate=0.5,
epochs=30,
layers=layers)
print("\nLogistic Regression has been training..")
myMLPClassifier.train()
print("Done..")
# Do the recognizer
# Explicitly specify the test set to be evaluated
mlpPred = myMLPClassifier.evaluate()
#
# Report the result #
print("=========================")
evaluator = Evaluator()
#
# # print("Result of the stupid recognizer:")
# # evaluator.printComparison(data.testSet, stupidPred)
# evaluator.printAccuracy(data.test_set, stupidPred)
# #
# # print("\nResult of the Perceptron recognizer (on test set):")
# # evaluator.printComparison(data.testSet, perceptronPred)
# evaluator.printAccuracy(data.test_set, perceptronPred)
# #
# # print("\nResult of the Logistic Regression recognizer (on test set):")
# # evaluator.printComparison(data.testSet, perceptronPred)
# evaluator.printAccuracy(data.test_set, lrPred)
#
print("\nResult of the Multi-layer Perceptron recognizer (on test set):")
# evaluator.printComparison(data.testSet, perceptronPred)
evaluator.printAccuracy(data.test_set, mlpPred)
#
# # Draw
# plot = PerformancePlot("Logistic Regression")
# plot.draw_performance_epoch(myLRClassifier.performances,
# myLRClassifier.epochs)
# 3D Plot learning_rates + epochs -> accuracies
print("Creating 3D plot. This may take some minutes...")
learning_rate_sample_count = 5
epochs_sample_count = 20
xticks = np.logspace(-10.0,
0,
base=10,
num=learning_rate_sample_count,
endpoint=False)
accuracies = []
learning_rates = []
epoch_values = []
for i in itertools.product(range(learning_rate_sample_count)):
learning_rate = 100 / np.exp(i)
print("Calculating accuracy for: learning rate = %s" % (learning_rate))
myMLPClassifier = MultilayerPerceptron(data.training_set,
data.validation_set,
data.test_set,
learning_rate=learning_rate,
epochs=epochs_sample_count,
layers=layers)
epoch_accuracies = myMLPClassifier.train(False)
lrPred = myMLPClassifier.evaluate()
epoch_values.append([e for e in range(epochs_sample_count)])
learning_rates.append(
[learning_rate for _ in range(epochs_sample_count)])
accuracies.append(epoch_accuracies)
accuracies_merged = list(itertools.chain(*accuracies))
epochs_merged = list(itertools.chain(*epoch_values))
learning_rates_merged = list(itertools.chain(*learning_rates))
print(accuracies_merged)
print(epochs_merged)
print(learning_rates)
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(np.log10(learning_rates_merged), epochs_merged,
accuracies_merged)
ax.set_xlabel("Learning Rate")
ax.set_xticks(np.log10(xticks))
ax.set_xticklabels(xticks)
ax.set_ylabel('Epochs')
ax.set_zlabel('Accuracy')
plt.show()
def __init__(self,
train,
valid,
test,
layers=None,
inputWeights=None,
outputTask='classification',
outputActivation='softmax',
loss='bce',
learningRate=0.01,
epochs=50):
"""
A MNIST recognizer based on multi-layer perceptron algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
learningRate : float
epochs : positive int
performances: array of floats
"""
self.learningRate = learningRate
self.epochs = epochs
self.outputTask = outputTask # Either classification or regression
self.outputActivation = outputActivation
self.trainingSet = train
self.validationSet = valid
self.testSet = test
if loss == 'bce':
self.loss = BinaryCrossEntropyError()
elif loss == 'sse':
self.loss = SumSquaredError()
elif loss == 'mse':
self.loss = MeanSquaredError()
elif loss == 'different':
self.loss = DifferentError()
elif loss == 'absolute':
self.loss = AbsoluteError()
else:
raise ValueError('There is no predefined loss function ' +
'named ' + str)
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
self.layers = layers
# Build up the network from specific layers
self.layers = []
self.hiddenNeurons = 50
# Input layer
inputActivation = "sigmoid"
self.layers.append(
LogisticLayer(train.input.shape[1], 128, None, inputActivation,
False))
# Hidden layers - slightly increased accuracy with 2 hidden layers
hiddenActivation = "sigmoid"
self.layers.append(
LogisticLayer(128, self.hiddenNeurons, None, hiddenActivation,
False))
self.layers.append(
LogisticLayer(self.hiddenNeurons, self.hiddenNeurons, None,
hiddenActivation, False))
# Output layer
outputActivation = "softmax"
self.layers.append(
LogisticLayer(self.hiddenNeurons, 10, None, outputActivation,
True))
self.inputWeights = inputWeights
# add bias values ("1"s) at the beginning of all data sets
self.trainingSet.input = np.insert(self.trainingSet.input,
0,
1,
axis=1)
self.validationSet.input = np.insert(self.validationSet.input,
0,
1,
axis=1)
self.testSet.input = np.insert(self.testSet.input, 0, 1, axis=1)
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
"""
def __init__(self, train, valid, test, learningRate=0.01, epochs=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
self.layer = LogisticLayer(self.testSet.input.shape[1], 1, is_classifier_layer=True)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
performance = list()
for epoch in range(self.epochs):
if verbose:
print("Training epoch {0}/{1}.."
.format(epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validationSet.label,
self.evaluate(self.validationSet))
print("Accuracy on validation: {0:.2f}%"
.format(accuracy*100))
print("-----------------------------")
performance.append(accuracy * 100)
if verbose:
import matplotlib.pyplot as plt
plt.figure()
plt.xlabel("Epoch")
plt.ylabel("Accuracy (%)")
plt.xlim(1, self.epochs+1)
plt.ylim(0, 100)
plt.plot(np.arange(self.epochs)+1, performance, 'b-')
plt.show()
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
ind = np.arange(self.trainingSet.input.shape[0])
np.random.shuffle(ind)
for i in ind:
img, label = self.trainingSet.input[i], self.trainingSet.label[i]
self.layer.forward(img)
self.layer.computeDerivative(np.array([int(label)]), None)
self.layer.updateWeights(self.learningRate)
# if we want to do batch learning, accumulate the error
# and update the weight outside the loop
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
self.layer.forward(testInstance)
return self.layer.outp > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
def __init__(self, train, valid, test, learning_rate=0.01, epochs=50):
self.learning_rate = learning_rate
self.epochs = epochs
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
# Use a logistic layer as one-neuron classification (output) layer
self.layer = LogisticLayer(train.input.shape[1], 1,
is_classifier_layer=True)
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input, 0, 1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input, 0, 1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Run the training "epochs" times, print out the logs
for epoch in range(self.epochs):
if verbose:
print("Training epoch {0}/{1}.."
.format(epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validation_set.label,
self.evaluate(self.validation_set))
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances.append(accuracy)
print("Accuracy on validation: {0:.2f}%"
.format(accuracy * 100))
print("-----------------------------")
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
for img, label in zip(self.training_set.input,
self.training_set.label):
# Use LogisticLayer to do the job
# Feed it with inputs
# Do a forward pass to calculate the output and the error
self.layer.forward(img)
# Compute the derivatives w.r.t to the error
# Please note the treatment of nextDerivatives and nextWeights
# in case of an output layer
self.layer.computeDerivative(np.array(label - self.layer.outp),
np.ones(1))
# Update weights in the online learning fashion
self.layer.updateWeights(self.learning_rate)
def classify(self, test_instance):
"""Classify a single instance.
Parameters
----------
test_instance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Here you have to implement classification method given an instance
outp = self.layer.forward(test_instance)
return outp > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.test_set.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
def __del__(self):
# Remove the bias from input data
self.training_set.input = np.delete(self.training_set.input, 0, axis=1)
self.validation_set.input = np.delete(self.validation_set.input, 0,
axis=1)
self.test_set.input = np.delete(self.test_set.input, 0, axis=1)
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
layer : LogisticLayer
"""
def __init__(self, train, valid, test, learningRate=0.01, epochs=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
self.layer = LogisticLayer(self.trainingSet.input.shape[1], 1, learning_rate=learningRate)
def train(self, verbose=False, graph=False):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Data for graph
x = []
y = []
# Here you have to implement training method "epochs" times
# Please using LogisticLayer class
for epoch in range(0, self.epochs):
for label, data in zip(self.trainingSet.label, self.trainingSet.input):
self.classify(data)
self.layer.computeOutDerivative(label)
self.layer.updateWeights()
if verbose:
accuracy = accuracy_score(self.validationSet.label, self.evaluate(self.validationSet))
logging.info("New validation accuracy after epoch %i: %.1f%%", epoch + 1, accuracy * 100)
if graph:
y.append(accuracy)
if graph:
if len(y) == len(x):
# Do not evaluate twice
accuracy = accuracy_score(self.validationSet.label, self.evaluate(self.validationSet))
y.append(accuracy)
x.append(epoch)
if graph:
plt.plot(x, y)
plt.xlabel('epoch')
plt.ylabel('accuracy')
plt.title('Accuracy of different epochs')
plt.show()
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Here you have to implement classification method given an instance
self.layer.forward(testInstance)
return self.layer.getOutput() > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
def __init__(self, train, valid, test, learning_rate=0.01, epochs=50):
self.learning_rate = learning_rate
self.epochs = epochs
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
# Use a logistic layer as one-neuron classification (output) layer
self.layer = LogisticLayer(train.input.shape[1],
1,
is_classifier_layer=True)
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input,
0,
1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input,
0,
1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Run the training "epochs" times, print out the logs
for epoch in range(self.epochs):
if verbose:
print("Training epoch {0}/{1}..".format(
epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validation_set.label,
self.evaluate(self.validation_set))
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances.append(accuracy)
print("Accuracy on validation: {0:.2f}%".format(accuracy *
100))
print("-----------------------------")
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
for img, label in zip(self.training_set.input,
self.training_set.label):
# Use LogisticLayer to do the job
# Feed it with inputs
# Do a forward pass to calculate the output and the error
self.layer.forward(img)
# Compute the derivatives w.r.t to the error
# Please note the treatment of nextDerivatives and nextWeights
# in case of an output layer
self.layer.computeDerivative(np.array(label - self.layer.outp),
np.ones(1))
# Update weights in the online learning fashion
self.layer.updateWeights(self.learning_rate)
def classify(self, test_instance):
"""Classify a single instance.
Parameters
----------
test_instance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Here you have to implement classification method given an instance
outp = self.layer.forward(test_instance)
return outp > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.test_set.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
def __del__(self):
# Remove the bias from input data
self.training_set.input = np.delete(self.training_set.input, 0, axis=1)
self.validation_set.input = np.delete(self.validation_set.input,
0,
axis=1)
self.test_set.input = np.delete(self.test_set.input, 0, axis=1)
def main():
data = MNISTSeven("../data/mnist_seven.csv",
3000,
1000,
1000,
oneHot=False)
# myLRClassifier = LogisticRegression(data.trainingSet,
# data.validationSet,
# data.testSet,
# learningRate=0.005,
# epochs=30)
hidden_layers = [
LogisticLayer(128, 32, isClassifierLayer=True) for layer in range(1)
]
mlp = MultilayerPerceptron(data.trainingSet,
data.validationSet,
data.testSet,
hidden_layers,
learningRate=0.005,
epochs=30)
# Train the classifiers
#print("=========================")
print("Training...")
# print("\nLogistic Regression has been training..")
# myLRClassifier.train()
# print("Done..")
print("Training MLP...")
mlp.train()
print("Done.")
# Do the recognizer
# Explicitly specify the test set to be evaluated
# stupidPred = myStupidClassifier.evaluate()
# perceptronPred = myPerceptronClassifier.evaluate()
# lrPred = myLRClassifier.evaluate()
mlpPred = mlp.evaluate()
# Report the result
print("=========================")
evaluator = Evaluator()
# print("Result of the stupid recognizer:")
# # #evaluator.printComparison(data.testSet, stupidPred)
# # evaluator.printAccuracy(data.testSet, stupidPred)
# #
# # print("\nResult of the Perceptron recognizer:")
# # #evaluator.printComparison(data.testSet, perceptronPred)
# # evaluator.printAccuracy(data.testSet, perceptronPred)
# #
# # print("\nResult of the Logistic Regression recognizer:")
# # #evaluator.printComparison(data.testSet, lrPred)
# # evaluator.printAccuracy(data.testSet, lrPred)
print("Result of the MLP recognizer:")
evaluator.printComparison(data.testSet, mlpPred)
evaluator.printAccuracy(data.testSet, mlpPred)
# Draw
plot = PerformancePlot("Logistic Regression validation")
# plot.draw_performance_epoch(myLRClassifier.performances,
# myLRClassifier.epochs)
plot.draw_performance_epoch(mlp.performances, mlp.epochs)
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
"""
def __init__(self,
train,
valid,
test,
learningRate=0.01,
epochs=50,
activation='sigmoid',
error='mse'):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
# Initialize the weight vector with small values
self.weight = 0.01 * np.random.randn(1,
self.trainingSet.input.shape[1])
weight_Plus_bias = np.insert(self.weight, 0, 1, axis=1)
self.weight = weight_Plus_bias
# Choose the error function
self.errorString = error
self._initialize_error(error)
#initialize also the layer
self.layer = LogisticLayer(nIn=self.trainingSet.input.shape[1],
nOut=1,
activation='sigmoid',
weights=weight_Plus_bias)
def _initialize_error(self, error):
if error == 'absolute':
self.erf = erf.AbsoluteError()
elif error == 'mse':
self.erf = erf.MeanSquaredError()
elif error == 'sse':
self.erf = erf.SumSquaredError()
else:
raise ValueError(
'Cannot instantiate the requested error function:' + error +
'not available')
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
#from util.loss_functions import MeanSquaredError
#loss = MeanSquaredError()
learned = False
iteration = 0
accuracy = []
pl.ion()
input_Plus_bias = np.insert(self.trainingSet.input, 0, 1, axis=1)
#Train for some epochs if the error is not 0
while not learned:
totalError = 0
derivatives = []
hypothesis = np.array(
list(map(self.classify, self.trainingSet.input)))
totalError = self.erf.calculateError(
np.array(self.trainingSet.label), hypothesis)
#print("Error now is: %f", totalError)
if totalError != 0:
output = np.array([])
for i in range(0, self.trainingSet.input.shape[0]):
if i == 0:
output = self.layer.forward(self.trainingSet.input[i])
else:
np.append(output,
self.layer.forward(
self.trainingSet.input[i]),
axis=0)
dE_dy = self.erf.calculatePrime(
np.array(self.trainingSet.label), output)
# for only one neuron set the weight as [0,1]
dE_dx = self.layer.computeDerivative([dE_dy], [[0, 1]])
dE_dw = dE_dx * input_Plus_bias
self.layer.updateWeights(dE_dw)
iteration += 1
if verbose:
logging.info("Epoch: %i; Error: %f", iteration, totalError)
if totalError == 0 or iteration >= self.epochs:
learned = True
# accuracy.append(accuracy_score(self.trainingSet.label, hypothesis))
x = range(iteration)
# pl.xlabel(u"Epochs")
# pl.ylabel(u"Accuracy")
# pl.xlim(0, self.epochs)
# pl.ylim(0, 1.0)
# pl.plot(x, accuracy, 'k')
# pl.show()
# pl.pause(0.01)
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
return self.layer.forward(testInstance) >= 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
def updateWeights(self, grad):
self.weight += self.learningRate * grad
def fire(self, input):
# Look at how we change the activation function here!!!!
# Not Activation.sign as in the perceptron, but sigmoid
return Activation.sigmoid(np.dot(np.array(input), self.weight))
def __init__(self,
train,
valid,
test,
layers=None,
input_weights=None,
output_task='classification',
output_activation='softmax',
cost='crossentropy',
learning_rate=0.01,
epochs=50):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
self.learning_rate = learning_rate
self.epochs = epochs
self.output_task = output_task # Either classification or regression
self.output_activation = output_activation
self.cost = cost
# Should polish the loss_function a little bit more
self.error = CrossEntropyError
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
self.layers = layers
self.input_weights = input_weights
# Build up the network from specific layers
if layers is None:
self.layers = []
# First hidden layer
number_of_1st_hidden_layer = 100
self.layers.append(
LogisticLayer(train.input.shape[1],
number_of_1st_hidden_layer,
None,
activation="sigmoid",
is_classifier_layer=False))
# Output layer
self.layers.append(
LogisticLayer(number_of_1st_hidden_layer,
10,
None,
activation="softmax",
is_classifier_layer=True))
else:
self.layers = layers
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input,
0,
1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input,
0,
1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
class MultilayerPerceptron(Classifier):
"""
A multilayer perceptron used for classification
"""
def __init__(self, train, valid, test, layers=None, input_weights=None,
output_task='classification', output_activation='softmax',
cost='crossentropy', learning_rate=0.01, epochs=50, useDAE=None):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
self.learning_rate = learning_rate
self.epochs = epochs
self.output_task = output_task # Either classification or regression
self.output_activation = output_activation
self.cost = cost
# Should polish the loss_function a little bit more
self.error = CrossEntropyError
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
self.layers = layers
self.input_weights = input_weights
#self.input_weights = np.insert(self.input_weights, [0,0], [1,1])
self.input_weights = np.insert(input_weights, 0, 1, axis=0)
self.daeLayer = None
if input_weights is None:
self.input_weights = self.training_set.input
else:
self.daeLayer = LogisticLayer(self.training_set.input.shape[1],
self.input_weights.shape[1], self.input_weights,
activation="tanh",
is_classifier_layer=False)
# Build up the network from specific layers
if layers is None:
self.layers = []
# First hidden layer
number_of_1st_hidden_layer = 100
self.layers.append(LogisticLayer(self.input_weights.shape[1],
number_of_1st_hidden_layer, None,
activation="sigmoid",
is_classifier_layer=False))
# Output layer
self.layers.append(LogisticLayer(number_of_1st_hidden_layer,
10, None,
activation="softmax",
is_classifier_layer=True))
else:
self.layers = layers
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input, 0, 1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input, 0, 1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def _get_layer(self, layer_index):
return self.layers[layer_index]
def _get_input_layer(self):
return self._get_layer(0)
def _get_output_layer(self):
return self._get_layer(-1)
def _feed_forward(self, inp):
"""
Do feed forward through the layers of the network
Parameters
----------
inp : ndarray
a numpy array containing the input of the layer
"""
# Feed forward layer by layer
# The output of previous layer is the input of the next layer
if self.daeLayer is None:
last_layer_output = inp
else:
last_layer_output = self.daeLayer.forward(inp)
last_layer_output = np.insert(last_layer_output, 0, 1, axis=0)
for layer in self.layers:
last_layer_output = layer.forward(last_layer_output)
# Do not forget to add bias for every layer
last_layer_output = np.insert(last_layer_output, 0, 1, axis=0)
def _compute_error(self, target):
"""
Compute the total error of the network
Returns
-------
ndarray :
a numpy array (1,nOut) containing the output of the layer
"""
# Get output layer
output_layer = self._get_output_layer()
# Calculate the deltas of the output layer
output_layer.deltas = target - output_layer.outp
# Calculate deltas (error terms) backward except the output layer
for i in reversed(range(0, len(self.layers) - 1)):
current_layer = self._get_layer(i)
next_layer = self._get_layer(i+1)
next_weights = np.delete(next_layer.weights, 0, axis=0)
next_derivatives = next_layer.deltas
current_layer.computeDerivative(next_derivatives, next_weights.T)
def _update_weights(self):
"""
Update the weights of the layers by propagating back the error
"""
# Update the weights layer by layers
for layer in self.layers:
layer.updateWeights(self.learning_rate)
def train(self, verbose=True):
"""Train the Multi-layer Perceptrons
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Run the training "epochs" times, print out the logs
for epoch in range(self.epochs):
if verbose:
print("Training epoch {0}/{1}.."
.format(epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validation_set.label,
self.evaluate(self.validation_set))
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances.append(accuracy)
print("Accuracy on validation: {0:.2f}%"
.format(accuracy * 100))
print("-----------------------------")
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
#for img, label in zip(self.training_set.input,
# self.training_set.label):
# Use LogisticLayer to do the job
# Feed it with inputs
# Do a forward pass to calculate the output and the error
#self._feed_forward(img)
# Compute the derivatives w.r.t to the error
# Please note the treatment of nextDerivatives and nextWeights
# in case of an output layer
#self._compute_error(np.array(self._encode(label)))
#np.array(self._encode(label) - self._get_output_layer().outp),
for img, label in zip(self.training_set.input, self.training_set.label):
target = np.zeros(10)
target[label] = 1
self._feed_forward(img)
self._compute_error(target)
self._update_weights()
def classify(self, test_instance):
# Classify an instance given the model of the classifier
self._feed_forward(test_instance)
return np.argmax(self._get_output_layer().outp)
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.test_set.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
def __del__(self):
# Remove the bias from input data
self.training_set.input = np.delete(self.training_set.input, 0, axis=1)
self.validation_set.input = np.delete(self.validation_set.input, 0,
axis=1)
self.test_set.input = np.delete(self.test_set.input, 0, axis=1)
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
"""
def __init__(self, train, valid, test, learningRate=0.01, epochs=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
self.logisticLayer = LogisticLayer(len(self.trainingSet.input[0]),
1)
#None,
#'sigmoid',
#False)
#self.logisticLayer1 = LogisticLayer(len(self.trainingSet.input[0]),
# 16)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Here you have to implement training method "epochs" times
# Please using LogisticLayer class
for epoch in xrange(self.epochs):
if verbose:
print("Training epoch {0}/{1}.."
.format(epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validationSet.label,
self.evaluate(self.validationSet))
print("Accuracy on validation: {0:.2f}%"
.format(accuracy*100))
print("-----------------------------")
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
# for each training example do one forward pass (single layered neural network)
for img, label in zip(self.trainingSet.input, self.trainingSet.label):
#output1 = self.logisticLayer1.forward(img, False)
output = self.logisticLayer.forward(img, False)
# as this is the output layer: compute the output delta and pass it to layer
delta = (label - output[0]) * output[0] * (1 - output[0])
self.logisticLayer.computeDerivative([delta], None)
#self.logisticLayer1.computeDerivative(self.logisticLayer.deltas, self.logisticLayer.weights)
# online learning: updating weights after seeing 1 instance
# if we want to do batch learning, accumulate the error
# and update the weight outside the loop
# update the weights of the layer
self.logisticLayer.updateWeights(self.learningRate)
#self.logisticLayer1.updateWeights(self.learningRate)
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Here you have to implement classification method given an instance
#output1 = self.logisticLayer1.forward(testInstance, True)
output = self.logisticLayer.forward(testInstance, True)
# output is in interval [0, 1], generally if it is > 0.5 it is counted as True
return (output[0] > 0.5)
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
def __init__(self,
train,
valid,
test,
layers=None,
input_weights=None,
output_task='classification',
output_activation='softmax',
inputActivation='sigmoid',
cost='crossentropy',
learning_rate=0.01,
epochs=50,
learningRateReductionFactor=1.0,
layerNeurons=[10]):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
self.learning_rate = learning_rate
self.epochs = epochs
self.output_task = output_task # Either classification or regression
self.output_activation = output_activation
self.cost = cost
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
self.layers = layers
self.input_weights = input_weights
# activation function for the hidden layers
self.inputActivation = inputActivation
# reduction factor of learning rate per epoch
self.learningRateReductionFactor = learningRateReductionFactor
#######################
# CREATE LAYERS #
#######################
# Build up the network from specific layers
self.layers = []
# check for correct argument
if (len(layerNeurons) < 1):
raise ValueError(
'Error: layerNeurons must contain at least one layer with neurons!'
)
# if there is only one layer it is an output layer
if (len(layerNeurons) == 1):
self.layers.append(
LogisticLayer(train.input.shape[1], layerNeurons[0], None,
self.output_activation, True))
# if there are more than one layer
else:
# first layer (hidden layer)
self.layers.append(
LogisticLayer(train.input.shape[1], layerNeurons[0], None,
self.inputActivation, False))
# rest of the hidden layers
for i in xrange(1, len(layerNeurons) - 1):
self.layers.append(
LogisticLayer(layerNeurons[i - 1], layerNeurons[i], None,
self.inputActivation, False))
# output layer
self.layers.append(
LogisticLayer(layerNeurons[len(layerNeurons) - 2],
layerNeurons[len(layerNeurons) - 1], None,
self.output_activation, True))
# total number of output neurons
self.totalOutputs = layerNeurons[len(layerNeurons) - 1]
# total number of layers
self.totalLayers = len(self.layers)
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input,
0,
1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input,
0,
1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def __init__(self, train, valid, test, layers=None, input_weights=None,
output_task='classification', output_activation='softmax',
cost='crossentropy', learning_rate=0.01, epochs=50, useDAE=None):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
self.learning_rate = learning_rate
self.epochs = epochs
self.output_task = output_task # Either classification or regression
self.output_activation = output_activation
self.cost = cost
# Should polish the loss_function a little bit more
self.error = CrossEntropyError
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
self.layers = layers
self.input_weights = input_weights
#self.input_weights = np.insert(self.input_weights, [0,0], [1,1])
self.input_weights = np.insert(input_weights, 0, 1, axis=0)
self.daeLayer = None
if input_weights is None:
self.input_weights = self.training_set.input
else:
self.daeLayer = LogisticLayer(self.training_set.input.shape[1],
self.input_weights.shape[1], self.input_weights,
activation="tanh",
is_classifier_layer=False)
# Build up the network from specific layers
if layers is None:
self.layers = []
# First hidden layer
number_of_1st_hidden_layer = 100
self.layers.append(LogisticLayer(self.input_weights.shape[1],
number_of_1st_hidden_layer, None,
activation="sigmoid",
is_classifier_layer=False))
# Output layer
self.layers.append(LogisticLayer(number_of_1st_hidden_layer,
10, None,
activation="softmax",
is_classifier_layer=True))
else:
self.layers = layers
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input, 0, 1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input, 0, 1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def __init__(self,
train,
valid,
test,
layers=None,
input_weights=None,
output_task='classification',
output_activation='sigmoid',
cost='mse',
learning_rate=0.01,
epochs=50):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
layers: list
List of layers
input_weights: list
weight layer
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
self.learning_rate = learning_rate
self.epochs = epochs
self.output_task = output_task # Either classification or regression
self.output_activation = output_activation
self.cost_string = cost
self.cost = loss_functions.get_loss(cost)
print("Task: {}, Activation Function {}, Error Function: {}".format(
self.output_task, self.output_activation, self.cost_string))
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
self.layers = layers
self.input_weights = input_weights
if layers is None:
if output_task == 'classification':
self.layers = []
output_activation = "sigmoid"
self.layers.append(
LogisticLayer(train.input.shape[1],
10,
activation=output_activation,
is_classifier_layer=False))
self.layers.append(
LogisticLayer(10,
10,
activation=output_activation,
is_classifier_layer=False))
self.layers.append(
LogisticLayer(10,
1,
activation=output_activation,
is_classifier_layer=True))
elif output_task == 'classify_all':
self.layers = []
self.layers.append(
Layer(train.input.shape[1],
100,
activation='sigmoid',
is_classifier_layer=False))
self.layers.append(
Layer(100,
10,
activation='softmax',
is_classifier_layer=True))
else:
self.layers = layers
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input,
0,
1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input,
0,
1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
"""
def __init__(self, data, learningRate=0.01, epochs=50, hiddensize=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = data.trainingSet
self.validationSet = data.validationSet
self.testSet = data.testSet
self.data=data
self.layer=LogisticLayer(data.trainingSet.input.shape[1],hiddensize,learningRate)
# Initialize the weight vector with small values
self.weight = 0.01*np.random.randn(self.layer.size)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
from util.loss_functions import DifferentError
loss = DifferentError()
learned = False
iteration = 0
while not learned:
self.shuffle()
totalError = 0
for input, label in zip(self.trainingSet.input,
self.trainingSet.label):
# feedforward
inputarray = input.reshape(1,len(input))
layeroutput = self.layer.forward(inputarray)
output = self.fire(layeroutput)
# compute gradient of regression
delta=label - output
grad =delta * self.layer.output
# backpropagation
self.layer.computeDerivative(delta,self.weight)
#update all weights
self.updateWeights(grad)
self.layer.updateWeights()
# compute recognizing error, not BCE using validation data
for input, label in zip(self.validationSet.input,
self.validationSet.label):
predictedLabel = self.classify(input)
error = loss.calculateError(label, predictedLabel)
totalError += error
totalError = abs(totalError)
iteration += 1
if verbose:
logging.info("Epoch: %i; Error: %i", iteration, totalError)
if totalError == 0 or iteration >= self.epochs:
# stop criteria is reached
learned = True
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# feedforward
testInstance=testInstance.reshape(1,len(testInstance))
output = self.fire(self.layer.forward(testInstance))
return output > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just se map for all of the test
# set.
return list(map(self.classify, test))
def updateWeights(self, grad):
self.weight += (self.learningRate*grad).reshape(self.weight.size)
def fire(self, input):
# input (n,1)
return Activation.sigmoid(np.dot(self.weight,input))
def shuffle(self):
self.data.myshuffle()
self.trainingSet=self.data.trainingSet
self.validationSet=self.data.validationSet
self.testSet=self.data.testSet
