class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
"""
def __init__(self, train, valid, test, learningRate=0.01, epochs=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
self.layer = LogisticLayer(self.testSet.input.shape[1], 1, is_classifier_layer=True)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
performance = list()
for epoch in range(self.epochs):
if verbose:
print("Training epoch {0}/{1}.."
.format(epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validationSet.label,
self.evaluate(self.validationSet))
print("Accuracy on validation: {0:.2f}%"
.format(accuracy*100))
print("-----------------------------")
performance.append(accuracy * 100)
if verbose:
import matplotlib.pyplot as plt
plt.figure()
plt.xlabel("Epoch")
plt.ylabel("Accuracy (%)")
plt.xlim(1, self.epochs+1)
plt.ylim(0, 100)
plt.plot(np.arange(self.epochs)+1, performance, 'b-')
plt.show()
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
ind = np.arange(self.trainingSet.input.shape[0])
np.random.shuffle(ind)
for i in ind:
img, label = self.trainingSet.input[i], self.trainingSet.label[i]
self.layer.forward(img)
self.layer.computeDerivative(np.array([int(label)]), None)
self.layer.updateWeights(self.learningRate)
# if we want to do batch learning, accumulate the error
# and update the weight outside the loop
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
self.layer.forward(testInstance)
return self.layer.outp > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
def __init__(self, train, valid, test, learning_rate=0.01, epochs=50):
self.learning_rate = learning_rate
self.epochs = epochs
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
# Use a logistic layer as one-neuron classification (output) layer
self.layer = LogisticLayer(train.input.shape[1], 1,
is_classifier_layer=True)
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input, 0, 1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input, 0, 1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Run the training "epochs" times, print out the logs
for epoch in range(self.epochs):
if verbose:
print("Training epoch {0}/{1}.."
.format(epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validation_set.label,
self.evaluate(self.validation_set))
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances.append(accuracy)
print("Accuracy on validation: {0:.2f}%"
.format(accuracy * 100))
print("-----------------------------")
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
for img, label in zip(self.training_set.input,
self.training_set.label):
# Use LogisticLayer to do the job
# Feed it with inputs
# Do a forward pass to calculate the output and the error
self.layer.forward(img)
# Compute the derivatives w.r.t to the error
# Please note the treatment of nextDerivatives and nextWeights
# in case of an output layer
self.layer.computeDerivative(np.array(label - self.layer.outp),
np.ones(1))
# Update weights in the online learning fashion
self.layer.updateWeights(self.learning_rate)
def classify(self, test_instance):
"""Classify a single instance.
Parameters
----------
test_instance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Here you have to implement classification method given an instance
outp = self.layer.forward(test_instance)
return outp > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.test_set.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
def __del__(self):
# Remove the bias from input data
self.training_set.input = np.delete(self.training_set.input, 0, axis=1)
self.validation_set.input = np.delete(self.validation_set.input, 0,
axis=1)
self.test_set.input = np.delete(self.test_set.input, 0, axis=1)
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
def __init__(self, train, valid, test, learning_rate=0.01, epochs=50):
self.learning_rate = learning_rate
self.epochs = epochs
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
# Use a logistic layer as one-neuron classification (output) layer
self.layer = LogisticLayer(train.input.shape[1],
1,
is_classifier_layer=True)
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input,
0,
1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input,
0,
1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Run the training "epochs" times, print out the logs
for epoch in range(self.epochs):
if verbose:
print("Training epoch {0}/{1}..".format(
epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validation_set.label,
self.evaluate(self.validation_set))
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances.append(accuracy)
print("Accuracy on validation: {0:.2f}%".format(accuracy *
100))
print("-----------------------------")
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
for img, label in zip(self.training_set.input,
self.training_set.label):
# Use LogisticLayer to do the job
# Feed it with inputs
# Do a forward pass to calculate the output and the error
self.layer.forward(img)
# Compute the derivatives w.r.t to the error
# Please note the treatment of nextDerivatives and nextWeights
# in case of an output layer
self.layer.computeDerivative(np.array(label - self.layer.outp),
np.ones(1))
# Update weights in the online learning fashion
self.layer.updateWeights(self.learning_rate)
def classify(self, test_instance):
"""Classify a single instance.
Parameters
----------
test_instance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Here you have to implement classification method given an instance
outp = self.layer.forward(test_instance)
return outp > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.test_set.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
def __del__(self):
# Remove the bias from input data
self.training_set.input = np.delete(self.training_set.input, 0, axis=1)
self.validation_set.input = np.delete(self.validation_set.input,
0,
axis=1)
self.test_set.input = np.delete(self.test_set.input, 0, axis=1)
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
"""
def __init__(self, train, valid, test, learningRate=0.01, epochs=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
self.logisticLayer = LogisticLayer(len(self.trainingSet.input[0]),
1)
#None,
#'sigmoid',
#False)
#self.logisticLayer1 = LogisticLayer(len(self.trainingSet.input[0]),
# 16)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Here you have to implement training method "epochs" times
# Please using LogisticLayer class
for epoch in xrange(self.epochs):
if verbose:
print("Training epoch {0}/{1}.."
.format(epoch + 1, self.epochs))
self._train_one_epoch()
if verbose:
accuracy = accuracy_score(self.validationSet.label,
self.evaluate(self.validationSet))
print("Accuracy on validation: {0:.2f}%"
.format(accuracy*100))
print("-----------------------------")
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
# for each training example do one forward pass (single layered neural network)
for img, label in zip(self.trainingSet.input, self.trainingSet.label):
#output1 = self.logisticLayer1.forward(img, False)
output = self.logisticLayer.forward(img, False)
# as this is the output layer: compute the output delta and pass it to layer
delta = (label - output[0]) * output[0] * (1 - output[0])
self.logisticLayer.computeDerivative([delta], None)
#self.logisticLayer1.computeDerivative(self.logisticLayer.deltas, self.logisticLayer.weights)
# online learning: updating weights after seeing 1 instance
# if we want to do batch learning, accumulate the error
# and update the weight outside the loop
# update the weights of the layer
self.logisticLayer.updateWeights(self.learningRate)
#self.logisticLayer1.updateWeights(self.learningRate)
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Here you have to implement classification method given an instance
#output1 = self.logisticLayer1.forward(testInstance, True)
output = self.logisticLayer.forward(testInstance, True)
# output is in interval [0, 1], generally if it is > 0.5 it is counted as True
return (output[0] > 0.5)
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
"""
def __init__(self,
train,
valid,
test,
learningRate=0.01,
epochs=50,
activation='sigmoid',
error='mse'):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
# Initialize the weight vector with small values
self.weight = 0.01 * np.random.randn(1,
self.trainingSet.input.shape[1])
weight_Plus_bias = np.insert(self.weight, 0, 1, axis=1)
self.weight = weight_Plus_bias
# Choose the error function
self.errorString = error
self._initialize_error(error)
#initialize also the layer
self.layer = LogisticLayer(nIn=self.trainingSet.input.shape[1],
nOut=1,
activation='sigmoid',
weights=weight_Plus_bias)
def _initialize_error(self, error):
if error == 'absolute':
self.erf = erf.AbsoluteError()
elif error == 'mse':
self.erf = erf.MeanSquaredError()
elif error == 'sse':
self.erf = erf.SumSquaredError()
else:
raise ValueError(
'Cannot instantiate the requested error function:' + error +
'not available')
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
#from util.loss_functions import MeanSquaredError
#loss = MeanSquaredError()
learned = False
iteration = 0
accuracy = []
pl.ion()
input_Plus_bias = np.insert(self.trainingSet.input, 0, 1, axis=1)
#Train for some epochs if the error is not 0
while not learned:
totalError = 0
derivatives = []
hypothesis = np.array(
list(map(self.classify, self.trainingSet.input)))
totalError = self.erf.calculateError(
np.array(self.trainingSet.label), hypothesis)
#print("Error now is: %f", totalError)
if totalError != 0:
output = np.array([])
for i in range(0, self.trainingSet.input.shape[0]):
if i == 0:
output = self.layer.forward(self.trainingSet.input[i])
else:
np.append(output,
self.layer.forward(
self.trainingSet.input[i]),
axis=0)
dE_dy = self.erf.calculatePrime(
np.array(self.trainingSet.label), output)
# for only one neuron set the weight as [0,1]
dE_dx = self.layer.computeDerivative([dE_dy], [[0, 1]])
dE_dw = dE_dx * input_Plus_bias
self.layer.updateWeights(dE_dw)
iteration += 1
if verbose:
logging.info("Epoch: %i; Error: %f", iteration, totalError)
if totalError == 0 or iteration >= self.epochs:
learned = True
# accuracy.append(accuracy_score(self.trainingSet.label, hypothesis))
x = range(iteration)
# pl.xlabel(u"Epochs")
# pl.ylabel(u"Accuracy")
# pl.xlim(0, self.epochs)
# pl.ylim(0, 1.0)
# pl.plot(x, accuracy, 'k')
# pl.show()
# pl.pause(0.01)
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
return self.layer.forward(testInstance) >= 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just use map for all of the test
# set.
return list(map(self.classify, test))
def updateWeights(self, grad):
self.weight += self.learningRate * grad
def fire(self, input):
# Look at how we change the activation function here!!!!
# Not Activation.sign as in the perceptron, but sigmoid
return Activation.sigmoid(np.dot(np.array(input), self.weight))
class LogisticRegression(Classifier):
"""
A digit-7 recognizer based on logistic regression algorithm
Parameters
----------
train : list
valid : list
test : list
learningRate : float
epochs : positive int
Attributes
----------
trainingSet : list
validationSet : list
testSet : list
weight : list
learningRate : float
epochs : positive int
"""
def __init__(self, data, learningRate=0.01, epochs=50, hiddensize=50):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = data.trainingSet
self.validationSet = data.validationSet
self.testSet = data.testSet
self.data=data
self.layer=LogisticLayer(data.trainingSet.input.shape[1],hiddensize,learningRate)
# Initialize the weight vector with small values
self.weight = 0.01*np.random.randn(self.layer.size)
def train(self, verbose=True):
"""Train the Logistic Regression.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
from util.loss_functions import DifferentError
loss = DifferentError()
learned = False
iteration = 0
while not learned:
self.shuffle()
totalError = 0
for input, label in zip(self.trainingSet.input,
self.trainingSet.label):
# feedforward
inputarray = input.reshape(1,len(input))
layeroutput = self.layer.forward(inputarray)
output = self.fire(layeroutput)
# compute gradient of regression
delta=label - output
grad =delta * self.layer.output
# backpropagation
self.layer.computeDerivative(delta,self.weight)
#update all weights
self.updateWeights(grad)
self.layer.updateWeights()
# compute recognizing error, not BCE using validation data
for input, label in zip(self.validationSet.input,
self.validationSet.label):
predictedLabel = self.classify(input)
error = loss.calculateError(label, predictedLabel)
totalError += error
totalError = abs(totalError)
iteration += 1
if verbose:
logging.info("Epoch: %i; Error: %i", iteration, totalError)
if totalError == 0 or iteration >= self.epochs:
# stop criteria is reached
learned = True
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# feedforward
testInstance=testInstance.reshape(1,len(testInstance))
output = self.fire(self.layer.forward(testInstance))
return output > 0.5
def evaluate(self, test=None):
"""Evaluate a whole dataset.
Parameters
----------
test : the dataset to be classified
if no test data, the test set associated to the classifier will be used
Returns
-------
List:
List of classified decisions for the dataset's entries.
"""
if test is None:
test = self.testSet.input
# Once you can classify an instance, just se map for all of the test
# set.
return list(map(self.classify, test))
def updateWeights(self, grad):
self.weight += (self.learningRate*grad).reshape(self.weight.size)
def fire(self, input):
# input (n,1)
return Activation.sigmoid(np.dot(self.weight,input))
def shuffle(self):
self.data.myshuffle()
self.trainingSet=self.data.trainingSet
self.validationSet=self.data.validationSet
self.testSet=self.data.testSet
