def __init__(self, train, valid, test, learningRate=0.01, epochs=50, activation='sigmoid', error='mse'): self.learningRate = learningRate self.epochs = epochs self.trainingSet = train self.validationSet = valid self.testSet = test # Initialize the weight vector with small values self.weight = 0.01 * np.random.randn(1, self.trainingSet.input.shape[1]) weight_Plus_bias = np.insert(self.weight, 0, 1, axis=1) self.weight = weight_Plus_bias # Choose the error function self.errorString = error self._initialize_error(error) #initialize also the layer self.layer = LogisticLayer(nIn=self.trainingSet.input.shape[1], nOut=1, activation='sigmoid', weights=weight_Plus_bias)
def __init__(self, train, valid, test, learning_rate=0.01, epochs=50): self.learning_rate = learning_rate self.epochs = epochs self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] # Use a logistic layer as one-neuron classification (output) layer self.layer = LogisticLayer(train.input.shape[1], 1, is_classifier_layer=True) # add bias values ("1"s) at the beginning of all data sets self.training_set.input = np.insert(self.training_set.input, 0, 1, axis=1) self.validation_set.input = np.insert(self.validation_set.input, 0, 1, axis=1) self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def __init__(self, data, learningRate=0.01, epochs=50, hiddensize=50): self.learningRate = learningRate self.epochs = epochs self.trainingSet = data.trainingSet self.validationSet = data.validationSet self.testSet = data.testSet self.data=data self.layer=LogisticLayer(data.trainingSet.input.shape[1],hiddensize,learningRate) # Initialize the weight vector with small values self.weight = 0.01*np.random.randn(self.layer.size)
def __init__(self, train, valid, test, learningRate=0.01, epochs=50): self.learningRate = learningRate self.epochs = epochs self.trainingSet = train self.validationSet = valid self.testSet = test self.logisticLayer = LogisticLayer(len(self.trainingSet.input[0]), 1)
def __init__(self, train, valid, test, learningRate=0.01, epochs=50, loss='bce'): self.learningRate = learningRate self.epochs = epochs self.trainingSet = train self.validationSet = valid self.testSet = test if loss == 'bce': self.loss = BinaryCrossEntropyError() elif loss == 'sse': self.loss = SumSquaredError() elif loss == 'mse': self.loss = MeanSquaredError() elif loss == 'different': self.loss = DifferentError() elif loss == 'absolute': self.loss = AbsoluteError() else: raise ValueError('There is no predefined loss function ' + 'named ' + str) # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] # Use a logistic layer as one-neuron classification (output) layer self.layer = LogisticLayer(train.input.shape[1], 1, activation='sigmoid', isClassifierLayer=True) # add bias values ("1"s) at the beginning of all data sets self.trainingSet.input = np.insert(self.trainingSet.input, 0, 1, axis=1) self.validationSet.input = np.insert(self.validationSet.input, 0, 1, axis=1) self.testSet.input = np.insert(self.testSet.input, 0, 1, axis=1)
def _costructNetwork(self, netStruct, activationFunctions): prevSize = self.trainingSet.input.shape[1] - 1 for (size, func) in zip(netStruct, activationFunctions): self.layers.append(LogisticLayer(prevSize, size, None, func, False)) prevSize = size self.layers[-1].isClassifierLayer = True
def __init__(self, train, valid, test, learningRate=0.01, epochs=50): self.learningRate = learningRate self.epochs = epochs self.trainingSet = train self.validationSet = valid self.testSet = test self.layer = LogisticLayer(self.trainingSet.input.shape[1], 1, learning_rate=learningRate)
def main(args): hidden_layers = [ LogisticLayer(128, 128, isClassifierLayer=True) for layer in range(args.num_layers) ] data = MNISTSeven(args.dataset, 3000, 1000, 1000, oneHot=True) MLP = MultilayerPerceptron(data.trainingSet, data.validationSet, data.testSet, hidden_layers) MLP.train(verbose=True)
def __init__(self, train, valid, test, learning_rate=0.01, epochs=50): self.learning_rate = learning_rate self.epochs = epochs self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] # Use a logistic layer as one-neuron classification (output) layer self.layer = LogisticLayer(train.input.shape[1], 1, is_classifier_layer=True)
def main(): #data = MNISTSeven("../data/mnist_seven.csv", 3000, 1000, 1000, # one_hot=True, target_digit='7') # NOTE: # Comment out the MNISTSeven instantiation above and # uncomment the following to work with full MNIST task data = MNISTSeven("../data/mnist_seven.csv", 3000, 1000, 1000, one_hot=False) # # NOTE: # # Other 1-digit classifiers do not make sense now for comparison purpose # # So you should comment them out, let alone the MLP training and evaluation # # # Train the classifiers # # print("=========================") # print("Training..") # # # Stupid Classifier # myStupidClassifier = StupidRecognizer(data.training_set, # data.validation_set, # data.test_set) # # print("\nStupid Classifier has been training..") # myStupidClassifier.train() # print("Done..") # # Do the recognizer # # Explicitly specify the test set to be evaluated # stupidPred = myStupidClassifier.evaluate() # # # Perceptron # myPerceptronClassifier = Perceptron(data.training_set, # data.validation_set, # data.test_set, # learning_rate=0.005, # epochs=10) # # print("\nPerceptron has been training..") # myPerceptronClassifier.train() # print("Done..") # # Do the recognizer # # Explicitly specify the test set to be evaluated # perceptronPred = myPerceptronClassifier.evaluate() # # # Logistic Regression # myLRClassifier = LogisticRegression(data.training_set, # data.validation_set, # data.test_set, # learning_rate=0.005, # epochs=30) # # print("\nLogistic Regression has been training..") # myLRClassifier.train() # print("Done..") # # Do the recognizer # # Explicitly specify the test set to be evaluated # lrPred = myLRClassifier.evaluate() # Build up the network from specific layers # Here is an example of a MLP acting like the Logistic Regression layers = [] layers.append(LogisticLayer(784, 5, None, "sigmoid", True)) layers.append(LogisticLayer(5, 10, None, "softmax", False)) myMLPClassifier = MultilayerPerceptron(data.training_set, data.validation_set, data.test_set, learning_rate=0.5, epochs=30, layers=layers) print("\nLogistic Regression has been training..") myMLPClassifier.train() print("Done..") # Do the recognizer # Explicitly specify the test set to be evaluated mlpPred = myMLPClassifier.evaluate() # # Report the result # print("=========================") evaluator = Evaluator() # # # print("Result of the stupid recognizer:") # # evaluator.printComparison(data.testSet, stupidPred) # evaluator.printAccuracy(data.test_set, stupidPred) # # # # print("\nResult of the Perceptron recognizer (on test set):") # # evaluator.printComparison(data.testSet, perceptronPred) # evaluator.printAccuracy(data.test_set, perceptronPred) # # # # print("\nResult of the Logistic Regression recognizer (on test set):") # # evaluator.printComparison(data.testSet, perceptronPred) # evaluator.printAccuracy(data.test_set, lrPred) # print("\nResult of the Multi-layer Perceptron recognizer (on test set):") # evaluator.printComparison(data.testSet, perceptronPred) evaluator.printAccuracy(data.test_set, mlpPred) # # # Draw # plot = PerformancePlot("Logistic Regression") # plot.draw_performance_epoch(myLRClassifier.performances, # myLRClassifier.epochs) # 3D Plot learning_rates + epochs -> accuracies print("Creating 3D plot. This may take some minutes...") learning_rate_sample_count = 5 epochs_sample_count = 20 xticks = np.logspace(-10.0, 0, base=10, num=learning_rate_sample_count, endpoint=False) accuracies = [] learning_rates = [] epoch_values = [] for i in itertools.product(range(learning_rate_sample_count)): learning_rate = 100 / np.exp(i) print("Calculating accuracy for: learning rate = %s" % (learning_rate)) myMLPClassifier = MultilayerPerceptron(data.training_set, data.validation_set, data.test_set, learning_rate=learning_rate, epochs=epochs_sample_count, layers=layers) epoch_accuracies = myMLPClassifier.train(False) lrPred = myMLPClassifier.evaluate() epoch_values.append([e for e in range(epochs_sample_count)]) learning_rates.append( [learning_rate for _ in range(epochs_sample_count)]) accuracies.append(epoch_accuracies) accuracies_merged = list(itertools.chain(*accuracies)) epochs_merged = list(itertools.chain(*epoch_values)) learning_rates_merged = list(itertools.chain(*learning_rates)) print(accuracies_merged) print(epochs_merged) print(learning_rates) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.scatter(np.log10(learning_rates_merged), epochs_merged, accuracies_merged) ax.set_xlabel("Learning Rate") ax.set_xticks(np.log10(xticks)) ax.set_xticklabels(xticks) ax.set_ylabel('Epochs') ax.set_zlabel('Accuracy') plt.show()
def __init__(self, train, valid, test, layers=None, inputWeights=None, outputTask='classification', outputActivation='softmax', loss='bce', learningRate=0.01, epochs=50): """ A MNIST recognizer based on multi-layer perceptron algorithm Parameters ---------- train : list valid : list test : list learningRate : float epochs : positive int Attributes ---------- trainingSet : list validationSet : list testSet : list learningRate : float epochs : positive int performances: array of floats """ self.learningRate = learningRate self.epochs = epochs self.outputTask = outputTask # Either classification or regression self.outputActivation = outputActivation self.trainingSet = train self.validationSet = valid self.testSet = test if loss == 'bce': self.loss = BinaryCrossEntropyError() elif loss == 'sse': self.loss = SumSquaredError() elif loss == 'mse': self.loss = MeanSquaredError() elif loss == 'different': self.loss = DifferentError() elif loss == 'absolute': self.loss = AbsoluteError() else: raise ValueError('There is no predefined loss function ' + 'named ' + str) # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] self.layers = layers # Build up the network from specific layers self.layers = [] self.hiddenNeurons = 50 # Input layer inputActivation = "sigmoid" self.layers.append( LogisticLayer(train.input.shape[1], 128, None, inputActivation, False)) # Hidden layers - slightly increased accuracy with 2 hidden layers hiddenActivation = "sigmoid" self.layers.append( LogisticLayer(128, self.hiddenNeurons, None, hiddenActivation, False)) self.layers.append( LogisticLayer(self.hiddenNeurons, self.hiddenNeurons, None, hiddenActivation, False)) # Output layer outputActivation = "softmax" self.layers.append( LogisticLayer(self.hiddenNeurons, 10, None, outputActivation, True)) self.inputWeights = inputWeights # add bias values ("1"s) at the beginning of all data sets self.trainingSet.input = np.insert(self.trainingSet.input, 0, 1, axis=1) self.validationSet.input = np.insert(self.validationSet.input, 0, 1, axis=1) self.testSet.input = np.insert(self.testSet.input, 0, 1, axis=1)
class LogisticRegression(Classifier): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learningRate : float epochs : positive int Attributes ---------- trainingSet : list validationSet : list testSet : list weight : list learningRate : float epochs : positive int """ def __init__(self, train, valid, test, learningRate=0.01, epochs=50): self.learningRate = learningRate self.epochs = epochs self.trainingSet = train self.validationSet = valid self.testSet = test self.layer = LogisticLayer(self.testSet.input.shape[1], 1, is_classifier_layer=True) def train(self, verbose=True): """Train the Logistic Regression. Parameters ---------- verbose : boolean Print logging messages with validation accuracy if verbose is True. """ performance = list() for epoch in range(self.epochs): if verbose: print("Training epoch {0}/{1}.." .format(epoch + 1, self.epochs)) self._train_one_epoch() if verbose: accuracy = accuracy_score(self.validationSet.label, self.evaluate(self.validationSet)) print("Accuracy on validation: {0:.2f}%" .format(accuracy*100)) print("-----------------------------") performance.append(accuracy * 100) if verbose: import matplotlib.pyplot as plt plt.figure() plt.xlabel("Epoch") plt.ylabel("Accuracy (%)") plt.xlim(1, self.epochs+1) plt.ylim(0, 100) plt.plot(np.arange(self.epochs)+1, performance, 'b-') plt.show() def _train_one_epoch(self): """ Train one epoch, seeing all input instances """ ind = np.arange(self.trainingSet.input.shape[0]) np.random.shuffle(ind) for i in ind: img, label = self.trainingSet.input[i], self.trainingSet.label[i] self.layer.forward(img) self.layer.computeDerivative(np.array([int(label)]), None) self.layer.updateWeights(self.learningRate) # if we want to do batch learning, accumulate the error # and update the weight outside the loop def classify(self, testInstance): """Classify a single instance. Parameters ---------- testInstance : list of floats Returns ------- bool : True if the testInstance is recognized as a 7, False otherwise. """ self.layer.forward(testInstance) return self.layer.outp > 0.5 def evaluate(self, test=None): """Evaluate a whole dataset. Parameters ---------- test : the dataset to be classified if no test data, the test set associated to the classifier will be used Returns ------- List: List of classified decisions for the dataset's entries. """ if test is None: test = self.testSet.input # Once you can classify an instance, just use map for all of the test # set. return list(map(self.classify, test))
class LogisticRegression(Classifier): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learning_rate : float epochs : positive int Attributes ---------- training_set : list validation_set : list test_set : list learning_rate : float epochs : positive int performances: array of floats """ def __init__(self, train, valid, test, learning_rate=0.01, epochs=50): self.learning_rate = learning_rate self.epochs = epochs self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] # Use a logistic layer as one-neuron classification (output) layer self.layer = LogisticLayer(train.input.shape[1], 1, is_classifier_layer=True) # add bias values ("1"s) at the beginning of all data sets self.training_set.input = np.insert(self.training_set.input, 0, 1, axis=1) self.validation_set.input = np.insert(self.validation_set.input, 0, 1, axis=1) self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1) def train(self, verbose=True): """Train the Logistic Regression. Parameters ---------- verbose : boolean Print logging messages with validation accuracy if verbose is True. """ # Run the training "epochs" times, print out the logs for epoch in range(self.epochs): if verbose: print("Training epoch {0}/{1}.." .format(epoch + 1, self.epochs)) self._train_one_epoch() if verbose: accuracy = accuracy_score(self.validation_set.label, self.evaluate(self.validation_set)) # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances.append(accuracy) print("Accuracy on validation: {0:.2f}%" .format(accuracy * 100)) print("-----------------------------") def _train_one_epoch(self): """ Train one epoch, seeing all input instances """ for img, label in zip(self.training_set.input, self.training_set.label): # Use LogisticLayer to do the job # Feed it with inputs # Do a forward pass to calculate the output and the error self.layer.forward(img) # Compute the derivatives w.r.t to the error # Please note the treatment of nextDerivatives and nextWeights # in case of an output layer self.layer.computeDerivative(np.array(label - self.layer.outp), np.ones(1)) # Update weights in the online learning fashion self.layer.updateWeights(self.learning_rate) def classify(self, test_instance): """Classify a single instance. Parameters ---------- test_instance : list of floats Returns ------- bool : True if the testInstance is recognized as a 7, False otherwise. """ # Here you have to implement classification method given an instance outp = self.layer.forward(test_instance) return outp > 0.5 def evaluate(self, test=None): """Evaluate a whole dataset. Parameters ---------- test : the dataset to be classified if no test data, the test set associated to the classifier will be used Returns ------- List: List of classified decisions for the dataset's entries. """ if test is None: test = self.test_set.input # Once you can classify an instance, just use map for all of the test # set. return list(map(self.classify, test)) def __del__(self): # Remove the bias from input data self.training_set.input = np.delete(self.training_set.input, 0, axis=1) self.validation_set.input = np.delete(self.validation_set.input, 0, axis=1) self.test_set.input = np.delete(self.test_set.input, 0, axis=1)
class LogisticRegression(Classifier): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learningRate : float epochs : positive int Attributes ---------- trainingSet : list validationSet : list testSet : list weight : list learningRate : float epochs : positive int layer : LogisticLayer """ def __init__(self, train, valid, test, learningRate=0.01, epochs=50): self.learningRate = learningRate self.epochs = epochs self.trainingSet = train self.validationSet = valid self.testSet = test self.layer = LogisticLayer(self.trainingSet.input.shape[1], 1, learning_rate=learningRate) def train(self, verbose=False, graph=False): """Train the Logistic Regression. Parameters ---------- verbose : boolean Print logging messages with validation accuracy if verbose is True. """ # Data for graph x = [] y = [] # Here you have to implement training method "epochs" times # Please using LogisticLayer class for epoch in range(0, self.epochs): for label, data in zip(self.trainingSet.label, self.trainingSet.input): self.classify(data) self.layer.computeOutDerivative(label) self.layer.updateWeights() if verbose: accuracy = accuracy_score(self.validationSet.label, self.evaluate(self.validationSet)) logging.info("New validation accuracy after epoch %i: %.1f%%", epoch + 1, accuracy * 100) if graph: y.append(accuracy) if graph: if len(y) == len(x): # Do not evaluate twice accuracy = accuracy_score(self.validationSet.label, self.evaluate(self.validationSet)) y.append(accuracy) x.append(epoch) if graph: plt.plot(x, y) plt.xlabel('epoch') plt.ylabel('accuracy') plt.title('Accuracy of different epochs') plt.show() def classify(self, testInstance): """Classify a single instance. Parameters ---------- testInstance : list of floats Returns ------- bool : True if the testInstance is recognized as a 7, False otherwise. """ # Here you have to implement classification method given an instance self.layer.forward(testInstance) return self.layer.getOutput() > 0.5 def evaluate(self, test=None): """Evaluate a whole dataset. Parameters ---------- test : the dataset to be classified if no test data, the test set associated to the classifier will be used Returns ------- List: List of classified decisions for the dataset's entries. """ if test is None: test = self.testSet.input # Once you can classify an instance, just use map for all of the test # set. return list(map(self.classify, test))
class LogisticRegression(Classifier): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learning_rate : float epochs : positive int Attributes ---------- training_set : list validation_set : list test_set : list learning_rate : float epochs : positive int performances: array of floats """ def __init__(self, train, valid, test, learning_rate=0.01, epochs=50): self.learning_rate = learning_rate self.epochs = epochs self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] # Use a logistic layer as one-neuron classification (output) layer self.layer = LogisticLayer(train.input.shape[1], 1, is_classifier_layer=True) # add bias values ("1"s) at the beginning of all data sets self.training_set.input = np.insert(self.training_set.input, 0, 1, axis=1) self.validation_set.input = np.insert(self.validation_set.input, 0, 1, axis=1) self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1) def train(self, verbose=True): """Train the Logistic Regression. Parameters ---------- verbose : boolean Print logging messages with validation accuracy if verbose is True. """ # Run the training "epochs" times, print out the logs for epoch in range(self.epochs): if verbose: print("Training epoch {0}/{1}..".format( epoch + 1, self.epochs)) self._train_one_epoch() if verbose: accuracy = accuracy_score(self.validation_set.label, self.evaluate(self.validation_set)) # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances.append(accuracy) print("Accuracy on validation: {0:.2f}%".format(accuracy * 100)) print("-----------------------------") def _train_one_epoch(self): """ Train one epoch, seeing all input instances """ for img, label in zip(self.training_set.input, self.training_set.label): # Use LogisticLayer to do the job # Feed it with inputs # Do a forward pass to calculate the output and the error self.layer.forward(img) # Compute the derivatives w.r.t to the error # Please note the treatment of nextDerivatives and nextWeights # in case of an output layer self.layer.computeDerivative(np.array(label - self.layer.outp), np.ones(1)) # Update weights in the online learning fashion self.layer.updateWeights(self.learning_rate) def classify(self, test_instance): """Classify a single instance. Parameters ---------- test_instance : list of floats Returns ------- bool : True if the testInstance is recognized as a 7, False otherwise. """ # Here you have to implement classification method given an instance outp = self.layer.forward(test_instance) return outp > 0.5 def evaluate(self, test=None): """Evaluate a whole dataset. Parameters ---------- test : the dataset to be classified if no test data, the test set associated to the classifier will be used Returns ------- List: List of classified decisions for the dataset's entries. """ if test is None: test = self.test_set.input # Once you can classify an instance, just use map for all of the test # set. return list(map(self.classify, test)) def __del__(self): # Remove the bias from input data self.training_set.input = np.delete(self.training_set.input, 0, axis=1) self.validation_set.input = np.delete(self.validation_set.input, 0, axis=1) self.test_set.input = np.delete(self.test_set.input, 0, axis=1)
def main(): data = MNISTSeven("../data/mnist_seven.csv", 3000, 1000, 1000, oneHot=False) # myLRClassifier = LogisticRegression(data.trainingSet, # data.validationSet, # data.testSet, # learningRate=0.005, # epochs=30) hidden_layers = [ LogisticLayer(128, 32, isClassifierLayer=True) for layer in range(1) ] mlp = MultilayerPerceptron(data.trainingSet, data.validationSet, data.testSet, hidden_layers, learningRate=0.005, epochs=30) # Train the classifiers #print("=========================") print("Training...") # print("\nLogistic Regression has been training..") # myLRClassifier.train() # print("Done..") print("Training MLP...") mlp.train() print("Done.") # Do the recognizer # Explicitly specify the test set to be evaluated # stupidPred = myStupidClassifier.evaluate() # perceptronPred = myPerceptronClassifier.evaluate() # lrPred = myLRClassifier.evaluate() mlpPred = mlp.evaluate() # Report the result print("=========================") evaluator = Evaluator() # print("Result of the stupid recognizer:") # # #evaluator.printComparison(data.testSet, stupidPred) # # evaluator.printAccuracy(data.testSet, stupidPred) # # # # print("\nResult of the Perceptron recognizer:") # # #evaluator.printComparison(data.testSet, perceptronPred) # # evaluator.printAccuracy(data.testSet, perceptronPred) # # # # print("\nResult of the Logistic Regression recognizer:") # # #evaluator.printComparison(data.testSet, lrPred) # # evaluator.printAccuracy(data.testSet, lrPred) print("Result of the MLP recognizer:") evaluator.printComparison(data.testSet, mlpPred) evaluator.printAccuracy(data.testSet, mlpPred) # Draw plot = PerformancePlot("Logistic Regression validation") # plot.draw_performance_epoch(myLRClassifier.performances, # myLRClassifier.epochs) plot.draw_performance_epoch(mlp.performances, mlp.epochs)
class LogisticRegression(Classifier): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learningRate : float epochs : positive int Attributes ---------- trainingSet : list validationSet : list testSet : list weight : list learningRate : float epochs : positive int """ def __init__(self, train, valid, test, learningRate=0.01, epochs=50, activation='sigmoid', error='mse'): self.learningRate = learningRate self.epochs = epochs self.trainingSet = train self.validationSet = valid self.testSet = test # Initialize the weight vector with small values self.weight = 0.01 * np.random.randn(1, self.trainingSet.input.shape[1]) weight_Plus_bias = np.insert(self.weight, 0, 1, axis=1) self.weight = weight_Plus_bias # Choose the error function self.errorString = error self._initialize_error(error) #initialize also the layer self.layer = LogisticLayer(nIn=self.trainingSet.input.shape[1], nOut=1, activation='sigmoid', weights=weight_Plus_bias) def _initialize_error(self, error): if error == 'absolute': self.erf = erf.AbsoluteError() elif error == 'mse': self.erf = erf.MeanSquaredError() elif error == 'sse': self.erf = erf.SumSquaredError() else: raise ValueError( 'Cannot instantiate the requested error function:' + error + 'not available') def train(self, verbose=True): """Train the Logistic Regression. Parameters ---------- verbose : boolean Print logging messages with validation accuracy if verbose is True. """ #from util.loss_functions import MeanSquaredError #loss = MeanSquaredError() learned = False iteration = 0 accuracy = [] pl.ion() input_Plus_bias = np.insert(self.trainingSet.input, 0, 1, axis=1) #Train for some epochs if the error is not 0 while not learned: totalError = 0 derivatives = [] hypothesis = np.array( list(map(self.classify, self.trainingSet.input))) totalError = self.erf.calculateError( np.array(self.trainingSet.label), hypothesis) #print("Error now is: %f", totalError) if totalError != 0: output = np.array([]) for i in range(0, self.trainingSet.input.shape[0]): if i == 0: output = self.layer.forward(self.trainingSet.input[i]) else: np.append(output, self.layer.forward( self.trainingSet.input[i]), axis=0) dE_dy = self.erf.calculatePrime( np.array(self.trainingSet.label), output) # for only one neuron set the weight as [0,1] dE_dx = self.layer.computeDerivative([dE_dy], [[0, 1]]) dE_dw = dE_dx * input_Plus_bias self.layer.updateWeights(dE_dw) iteration += 1 if verbose: logging.info("Epoch: %i; Error: %f", iteration, totalError) if totalError == 0 or iteration >= self.epochs: learned = True # accuracy.append(accuracy_score(self.trainingSet.label, hypothesis)) x = range(iteration) # pl.xlabel(u"Epochs") # pl.ylabel(u"Accuracy") # pl.xlim(0, self.epochs) # pl.ylim(0, 1.0) # pl.plot(x, accuracy, 'k') # pl.show() # pl.pause(0.01) def classify(self, testInstance): """Classify a single instance. Parameters ---------- testInstance : list of floats Returns ------- bool : True if the testInstance is recognized as a 7, False otherwise. """ return self.layer.forward(testInstance) >= 0.5 def evaluate(self, test=None): """Evaluate a whole dataset. Parameters ---------- test : the dataset to be classified if no test data, the test set associated to the classifier will be used Returns ------- List: List of classified decisions for the dataset's entries. """ if test is None: test = self.testSet.input # Once you can classify an instance, just use map for all of the test # set. return list(map(self.classify, test)) def updateWeights(self, grad): self.weight += self.learningRate * grad def fire(self, input): # Look at how we change the activation function here!!!! # Not Activation.sign as in the perceptron, but sigmoid return Activation.sigmoid(np.dot(np.array(input), self.weight))
def __init__(self, train, valid, test, layers=None, input_weights=None, output_task='classification', output_activation='softmax', cost='crossentropy', learning_rate=0.01, epochs=50): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learning_rate : float epochs : positive int Attributes ---------- training_set : list validation_set : list test_set : list learning_rate : float epochs : positive int performances: array of floats """ self.learning_rate = learning_rate self.epochs = epochs self.output_task = output_task # Either classification or regression self.output_activation = output_activation self.cost = cost # Should polish the loss_function a little bit more self.error = CrossEntropyError self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] self.layers = layers self.input_weights = input_weights # Build up the network from specific layers if layers is None: self.layers = [] # First hidden layer number_of_1st_hidden_layer = 100 self.layers.append( LogisticLayer(train.input.shape[1], number_of_1st_hidden_layer, None, activation="sigmoid", is_classifier_layer=False)) # Output layer self.layers.append( LogisticLayer(number_of_1st_hidden_layer, 10, None, activation="softmax", is_classifier_layer=True)) else: self.layers = layers # add bias values ("1"s) at the beginning of all data sets self.training_set.input = np.insert(self.training_set.input, 0, 1, axis=1) self.validation_set.input = np.insert(self.validation_set.input, 0, 1, axis=1) self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
class MultilayerPerceptron(Classifier): """ A multilayer perceptron used for classification """ def __init__(self, train, valid, test, layers=None, input_weights=None, output_task='classification', output_activation='softmax', cost='crossentropy', learning_rate=0.01, epochs=50, useDAE=None): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learning_rate : float epochs : positive int Attributes ---------- training_set : list validation_set : list test_set : list learning_rate : float epochs : positive int performances: array of floats """ self.learning_rate = learning_rate self.epochs = epochs self.output_task = output_task # Either classification or regression self.output_activation = output_activation self.cost = cost # Should polish the loss_function a little bit more self.error = CrossEntropyError self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] self.layers = layers self.input_weights = input_weights #self.input_weights = np.insert(self.input_weights, [0,0], [1,1]) self.input_weights = np.insert(input_weights, 0, 1, axis=0) self.daeLayer = None if input_weights is None: self.input_weights = self.training_set.input else: self.daeLayer = LogisticLayer(self.training_set.input.shape[1], self.input_weights.shape[1], self.input_weights, activation="tanh", is_classifier_layer=False) # Build up the network from specific layers if layers is None: self.layers = [] # First hidden layer number_of_1st_hidden_layer = 100 self.layers.append(LogisticLayer(self.input_weights.shape[1], number_of_1st_hidden_layer, None, activation="sigmoid", is_classifier_layer=False)) # Output layer self.layers.append(LogisticLayer(number_of_1st_hidden_layer, 10, None, activation="softmax", is_classifier_layer=True)) else: self.layers = layers # add bias values ("1"s) at the beginning of all data sets self.training_set.input = np.insert(self.training_set.input, 0, 1, axis=1) self.validation_set.input = np.insert(self.validation_set.input, 0, 1, axis=1) self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1) def _get_layer(self, layer_index): return self.layers[layer_index] def _get_input_layer(self): return self._get_layer(0) def _get_output_layer(self): return self._get_layer(-1) def _feed_forward(self, inp): """ Do feed forward through the layers of the network Parameters ---------- inp : ndarray a numpy array containing the input of the layer """ # Feed forward layer by layer # The output of previous layer is the input of the next layer if self.daeLayer is None: last_layer_output = inp else: last_layer_output = self.daeLayer.forward(inp) last_layer_output = np.insert(last_layer_output, 0, 1, axis=0) for layer in self.layers: last_layer_output = layer.forward(last_layer_output) # Do not forget to add bias for every layer last_layer_output = np.insert(last_layer_output, 0, 1, axis=0) def _compute_error(self, target): """ Compute the total error of the network Returns ------- ndarray : a numpy array (1,nOut) containing the output of the layer """ # Get output layer output_layer = self._get_output_layer() # Calculate the deltas of the output layer output_layer.deltas = target - output_layer.outp # Calculate deltas (error terms) backward except the output layer for i in reversed(range(0, len(self.layers) - 1)): current_layer = self._get_layer(i) next_layer = self._get_layer(i+1) next_weights = np.delete(next_layer.weights, 0, axis=0) next_derivatives = next_layer.deltas current_layer.computeDerivative(next_derivatives, next_weights.T) def _update_weights(self): """ Update the weights of the layers by propagating back the error """ # Update the weights layer by layers for layer in self.layers: layer.updateWeights(self.learning_rate) def train(self, verbose=True): """Train the Multi-layer Perceptrons Parameters ---------- verbose : boolean Print logging messages with validation accuracy if verbose is True. """ # Run the training "epochs" times, print out the logs for epoch in range(self.epochs): if verbose: print("Training epoch {0}/{1}.." .format(epoch + 1, self.epochs)) self._train_one_epoch() if verbose: accuracy = accuracy_score(self.validation_set.label, self.evaluate(self.validation_set)) # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances.append(accuracy) print("Accuracy on validation: {0:.2f}%" .format(accuracy * 100)) print("-----------------------------") def _train_one_epoch(self): """ Train one epoch, seeing all input instances """ #for img, label in zip(self.training_set.input, # self.training_set.label): # Use LogisticLayer to do the job # Feed it with inputs # Do a forward pass to calculate the output and the error #self._feed_forward(img) # Compute the derivatives w.r.t to the error # Please note the treatment of nextDerivatives and nextWeights # in case of an output layer #self._compute_error(np.array(self._encode(label))) #np.array(self._encode(label) - self._get_output_layer().outp), for img, label in zip(self.training_set.input, self.training_set.label): target = np.zeros(10) target[label] = 1 self._feed_forward(img) self._compute_error(target) self._update_weights() def classify(self, test_instance): # Classify an instance given the model of the classifier self._feed_forward(test_instance) return np.argmax(self._get_output_layer().outp) def evaluate(self, test=None): """Evaluate a whole dataset. Parameters ---------- test : the dataset to be classified if no test data, the test set associated to the classifier will be used Returns ------- List: List of classified decisions for the dataset's entries. """ if test is None: test = self.test_set.input # Once you can classify an instance, just use map for all of the test # set. return list(map(self.classify, test)) def __del__(self): # Remove the bias from input data self.training_set.input = np.delete(self.training_set.input, 0, axis=1) self.validation_set.input = np.delete(self.validation_set.input, 0, axis=1) self.test_set.input = np.delete(self.test_set.input, 0, axis=1)
class LogisticRegression(Classifier): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learningRate : float epochs : positive int Attributes ---------- trainingSet : list validationSet : list testSet : list weight : list learningRate : float epochs : positive int """ def __init__(self, train, valid, test, learningRate=0.01, epochs=50): self.learningRate = learningRate self.epochs = epochs self.trainingSet = train self.validationSet = valid self.testSet = test self.logisticLayer = LogisticLayer(len(self.trainingSet.input[0]), 1) #None, #'sigmoid', #False) #self.logisticLayer1 = LogisticLayer(len(self.trainingSet.input[0]), # 16) def train(self, verbose=True): """Train the Logistic Regression. Parameters ---------- verbose : boolean Print logging messages with validation accuracy if verbose is True. """ # Here you have to implement training method "epochs" times # Please using LogisticLayer class for epoch in xrange(self.epochs): if verbose: print("Training epoch {0}/{1}.." .format(epoch + 1, self.epochs)) self._train_one_epoch() if verbose: accuracy = accuracy_score(self.validationSet.label, self.evaluate(self.validationSet)) print("Accuracy on validation: {0:.2f}%" .format(accuracy*100)) print("-----------------------------") def _train_one_epoch(self): """ Train one epoch, seeing all input instances """ # for each training example do one forward pass (single layered neural network) for img, label in zip(self.trainingSet.input, self.trainingSet.label): #output1 = self.logisticLayer1.forward(img, False) output = self.logisticLayer.forward(img, False) # as this is the output layer: compute the output delta and pass it to layer delta = (label - output[0]) * output[0] * (1 - output[0]) self.logisticLayer.computeDerivative([delta], None) #self.logisticLayer1.computeDerivative(self.logisticLayer.deltas, self.logisticLayer.weights) # online learning: updating weights after seeing 1 instance # if we want to do batch learning, accumulate the error # and update the weight outside the loop # update the weights of the layer self.logisticLayer.updateWeights(self.learningRate) #self.logisticLayer1.updateWeights(self.learningRate) def classify(self, testInstance): """Classify a single instance. Parameters ---------- testInstance : list of floats Returns ------- bool : True if the testInstance is recognized as a 7, False otherwise. """ # Here you have to implement classification method given an instance #output1 = self.logisticLayer1.forward(testInstance, True) output = self.logisticLayer.forward(testInstance, True) # output is in interval [0, 1], generally if it is > 0.5 it is counted as True return (output[0] > 0.5) def evaluate(self, test=None): """Evaluate a whole dataset. Parameters ---------- test : the dataset to be classified if no test data, the test set associated to the classifier will be used Returns ------- List: List of classified decisions for the dataset's entries. """ if test is None: test = self.testSet.input # Once you can classify an instance, just use map for all of the test # set. return list(map(self.classify, test))
def __init__(self, train, valid, test, layers=None, input_weights=None, output_task='classification', output_activation='softmax', inputActivation='sigmoid', cost='crossentropy', learning_rate=0.01, epochs=50, learningRateReductionFactor=1.0, layerNeurons=[10]): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learning_rate : float epochs : positive int Attributes ---------- training_set : list validation_set : list test_set : list learning_rate : float epochs : positive int performances: array of floats """ self.learning_rate = learning_rate self.epochs = epochs self.output_task = output_task # Either classification or regression self.output_activation = output_activation self.cost = cost self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] self.layers = layers self.input_weights = input_weights # activation function for the hidden layers self.inputActivation = inputActivation # reduction factor of learning rate per epoch self.learningRateReductionFactor = learningRateReductionFactor ####################### # CREATE LAYERS # ####################### # Build up the network from specific layers self.layers = [] # check for correct argument if (len(layerNeurons) < 1): raise ValueError( 'Error: layerNeurons must contain at least one layer with neurons!' ) # if there is only one layer it is an output layer if (len(layerNeurons) == 1): self.layers.append( LogisticLayer(train.input.shape[1], layerNeurons[0], None, self.output_activation, True)) # if there are more than one layer else: # first layer (hidden layer) self.layers.append( LogisticLayer(train.input.shape[1], layerNeurons[0], None, self.inputActivation, False)) # rest of the hidden layers for i in xrange(1, len(layerNeurons) - 1): self.layers.append( LogisticLayer(layerNeurons[i - 1], layerNeurons[i], None, self.inputActivation, False)) # output layer self.layers.append( LogisticLayer(layerNeurons[len(layerNeurons) - 2], layerNeurons[len(layerNeurons) - 1], None, self.output_activation, True)) # total number of output neurons self.totalOutputs = layerNeurons[len(layerNeurons) - 1] # total number of layers self.totalLayers = len(self.layers) # add bias values ("1"s) at the beginning of all data sets self.training_set.input = np.insert(self.training_set.input, 0, 1, axis=1) self.validation_set.input = np.insert(self.validation_set.input, 0, 1, axis=1) self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def __init__(self, train, valid, test, layers=None, input_weights=None, output_task='classification', output_activation='softmax', cost='crossentropy', learning_rate=0.01, epochs=50, useDAE=None): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learning_rate : float epochs : positive int Attributes ---------- training_set : list validation_set : list test_set : list learning_rate : float epochs : positive int performances: array of floats """ self.learning_rate = learning_rate self.epochs = epochs self.output_task = output_task # Either classification or regression self.output_activation = output_activation self.cost = cost # Should polish the loss_function a little bit more self.error = CrossEntropyError self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] self.layers = layers self.input_weights = input_weights #self.input_weights = np.insert(self.input_weights, [0,0], [1,1]) self.input_weights = np.insert(input_weights, 0, 1, axis=0) self.daeLayer = None if input_weights is None: self.input_weights = self.training_set.input else: self.daeLayer = LogisticLayer(self.training_set.input.shape[1], self.input_weights.shape[1], self.input_weights, activation="tanh", is_classifier_layer=False) # Build up the network from specific layers if layers is None: self.layers = [] # First hidden layer number_of_1st_hidden_layer = 100 self.layers.append(LogisticLayer(self.input_weights.shape[1], number_of_1st_hidden_layer, None, activation="sigmoid", is_classifier_layer=False)) # Output layer self.layers.append(LogisticLayer(number_of_1st_hidden_layer, 10, None, activation="softmax", is_classifier_layer=True)) else: self.layers = layers # add bias values ("1"s) at the beginning of all data sets self.training_set.input = np.insert(self.training_set.input, 0, 1, axis=1) self.validation_set.input = np.insert(self.validation_set.input, 0, 1, axis=1) self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def __init__(self, train, valid, test, layers=None, input_weights=None, output_task='classification', output_activation='sigmoid', cost='mse', learning_rate=0.01, epochs=50): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list layers: list List of layers input_weights: list weight layer learning_rate : float epochs : positive int Attributes ---------- training_set : list validation_set : list test_set : list learning_rate : float epochs : positive int performances: array of floats """ self.learning_rate = learning_rate self.epochs = epochs self.output_task = output_task # Either classification or regression self.output_activation = output_activation self.cost_string = cost self.cost = loss_functions.get_loss(cost) print("Task: {}, Activation Function {}, Error Function: {}".format( self.output_task, self.output_activation, self.cost_string)) self.training_set = train self.validation_set = valid self.test_set = test # Record the performance of each epoch for later usages # e.g. plotting, reporting.. self.performances = [] self.layers = layers self.input_weights = input_weights if layers is None: if output_task == 'classification': self.layers = [] output_activation = "sigmoid" self.layers.append( LogisticLayer(train.input.shape[1], 10, activation=output_activation, is_classifier_layer=False)) self.layers.append( LogisticLayer(10, 10, activation=output_activation, is_classifier_layer=False)) self.layers.append( LogisticLayer(10, 1, activation=output_activation, is_classifier_layer=True)) elif output_task == 'classify_all': self.layers = [] self.layers.append( Layer(train.input.shape[1], 100, activation='sigmoid', is_classifier_layer=False)) self.layers.append( Layer(100, 10, activation='softmax', is_classifier_layer=True)) else: self.layers = layers # add bias values ("1"s) at the beginning of all data sets self.training_set.input = np.insert(self.training_set.input, 0, 1, axis=1) self.validation_set.input = np.insert(self.validation_set.input, 0, 1, axis=1) self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
class LogisticRegression(Classifier): """ A digit-7 recognizer based on logistic regression algorithm Parameters ---------- train : list valid : list test : list learningRate : float epochs : positive int Attributes ---------- trainingSet : list validationSet : list testSet : list weight : list learningRate : float epochs : positive int """ def __init__(self, data, learningRate=0.01, epochs=50, hiddensize=50): self.learningRate = learningRate self.epochs = epochs self.trainingSet = data.trainingSet self.validationSet = data.validationSet self.testSet = data.testSet self.data=data self.layer=LogisticLayer(data.trainingSet.input.shape[1],hiddensize,learningRate) # Initialize the weight vector with small values self.weight = 0.01*np.random.randn(self.layer.size) def train(self, verbose=True): """Train the Logistic Regression. Parameters ---------- verbose : boolean Print logging messages with validation accuracy if verbose is True. """ from util.loss_functions import DifferentError loss = DifferentError() learned = False iteration = 0 while not learned: self.shuffle() totalError = 0 for input, label in zip(self.trainingSet.input, self.trainingSet.label): # feedforward inputarray = input.reshape(1,len(input)) layeroutput = self.layer.forward(inputarray) output = self.fire(layeroutput) # compute gradient of regression delta=label - output grad =delta * self.layer.output # backpropagation self.layer.computeDerivative(delta,self.weight) #update all weights self.updateWeights(grad) self.layer.updateWeights() # compute recognizing error, not BCE using validation data for input, label in zip(self.validationSet.input, self.validationSet.label): predictedLabel = self.classify(input) error = loss.calculateError(label, predictedLabel) totalError += error totalError = abs(totalError) iteration += 1 if verbose: logging.info("Epoch: %i; Error: %i", iteration, totalError) if totalError == 0 or iteration >= self.epochs: # stop criteria is reached learned = True def classify(self, testInstance): """Classify a single instance. Parameters ---------- testInstance : list of floats Returns ------- bool : True if the testInstance is recognized as a 7, False otherwise. """ # feedforward testInstance=testInstance.reshape(1,len(testInstance)) output = self.fire(self.layer.forward(testInstance)) return output > 0.5 def evaluate(self, test=None): """Evaluate a whole dataset. Parameters ---------- test : the dataset to be classified if no test data, the test set associated to the classifier will be used Returns ------- List: List of classified decisions for the dataset's entries. """ if test is None: test = self.testSet.input # Once you can classify an instance, just se map for all of the test # set. return list(map(self.classify, test)) def updateWeights(self, grad): self.weight += (self.learningRate*grad).reshape(self.weight.size) def fire(self, input): # input (n,1) return Activation.sigmoid(np.dot(self.weight,input)) def shuffle(self): self.data.myshuffle() self.trainingSet=self.data.trainingSet self.validationSet=self.data.validationSet self.testSet=self.data.testSet