def __init__(self, nIn, nOut, weights=None,
activation='softmax', isClassifierLayer=True):
# Get activation function from string
# Notice the functional programming paradigms of Python + Numpy
self.activationString = activation
self.activation = Activation.getActivation(self.activationString)
self.activationPrime = Activation.getDerivative(self.activationString)
self.nIn = nIn
self.nOut = nOut
# Adding bias
self.input = np.ndarray((nIn+1, 1))
self.input[0] = 1
self.output = np.ndarray((nOut, 1))
self.delta = np.zeros((nOut, 1))
# You can have better initialization here
# wij means the weight from Input(j) to the Output(i)
if weights is None:
rns = np.random.RandomState(int(time.time()))
self.weights = rns.uniform(size=(nOut, nIn + 1))-0.5
else:
self.weights = weights
self.isClassifierLayer = isClassifierLayer
# Some handy properties of the layers
self.size = self.nOut
self.shape = self.weights.shape
def __init__(self,
n_in,
n_out,
weights=None,
activation='sigmoid',
is_classifier_layer=False):
# Get activation function from string
self.activation_string = activation
self.activation = Activation.get_activation(self.activation_string)
self.activation_derivative = Activation.get_derivative(
self.activation_string)
self.n_in = n_in
self.n_out = n_out
self.inp = np.ndarray((n_in + 1, 1))
self.inp[0] = 1
self.outp = np.ndarray((n_out, 1))
self.deltas = np.zeros((n_out, 1))
# You can have better initialization here
if weights is None:
self.weights = np.random.rand(n_in + 1, n_out) / 10
else:
assert (weights.shape == (n_in + 1, n_out))
self.weights = weights
self.is_classifier_layer = is_classifier_layer
# Some handy properties of the layers
self.size = self.n_out
self.shape = self.weights.shape
def __init__(self,
nIn,
nOut,
weights=None,
activation='sigmoid',
isClassifierLayer=False):
# Get activation function from string
self.activationString = activation
self.activation = Activation.getActivation(self.activationString)
self.activationDerivative = Activation.getDerivative(
self.activationString)
self.nIn = nIn
self.nOut = nOut
self.inp = np.ndarray((nIn + 1, 1))
# self.inp[0] = 1
self.outp = np.ndarray((nOut, 1))
self.deltas = np.zeros((nOut, 1))
# You can have better initialization here
if weights is None:
rns = np.random.RandomState(int(time.time()))
self.weights = rns.uniform(size=(nIn + 1, nOut)) - 0.5
else:
assert (weights.shape == (nIn + 1, nOut))
self.weights = weights
self.isClassifierLayer = isClassifierLayer
# Some handy properties of the layers
self.size = self.nOut
self.shape = self.weights.shape
def __init__(self, n_in, n_out, weights=None,
activation='sigmoid', is_classifier_layer=False):
# Get activation function from string
self.activation_string = activation
self.activation = Activation.get_activation(self.activation_string)
self.activation_derivative = Activation.get_derivative(
self.activation_string)
self.n_in = n_in
self.n_out = n_out
self.inp = np.ndarray(n_in + 1)
self.inp[0] = 1
self.outp = np.ndarray(n_out)
self.deltas = np.zeros(n_out)
# You can have better initialization here
if weights is None:
self.weights = np.random.rand(n_in + 1, n_out) / 10 - 0.05
# Adjust weights to zero mean
for i in range(n_out):
self.weights[:][i] -= (sum(self.weights[:][i]) / len(self.weights[:][i]))
else:
assert(weights.shape == (n_in + 1, n_out))
self.weights = weights
self.is_classifier_layer = is_classifier_layer
# Some handy properties of the layers
self.size = self.n_out
self.shape = self.weights.shape
def compute_output(self, input):
if len(input) != len(self.weights):
raise ValueError("MLPNeuron: Bad input dimensions: "
"Got vector of length {}, expected {}".format(
len(input), len(self.weights)))
weighted_sum = np.dot(input, self.weights) + self.bias
return Activation.getActivation(self.activation)(weighted_sum)
def __init__(self, nIn, nOut, weights=None,
activation='softmax', isClassifierLayer=True):
# Get activation function from string
# Notice the functional programming paradigms of Python + Numpy
self.activationString = activation
self.activation = Activation.getActivation(self.activationString)
self.nIn = nIn
self.nOut = nOut
self.input = np.ndarray((nIn+1, 1))
self.input[0] = 1
self.output = np.ndarray((nOut, 1))
self.delta = np.zeros((nOut, 1))
# You can have better initialization here
if weights is None:
rns = np.random.RandomState(int(time.time()))
self.weights = rns.uniform(size=(nOut, nIn + 1))-0.5
else:
self.weights = weights
self.isClassifierLayer = isClassifierLayer
# Some handy properties of the layers
self.size = self.nOut
self.shape = self.weights.shape
def __init__(self, n_in, n_out, weights=None,
activation='sigmoid', is_classifier_layer=False):
# Get activation function from string
self.activation_string = activation
self.activation = Activation.get_activation(self.activation_string)
self.n_in = n_in
self.n_out = n_out
self.inp = np.ndarray((n_in+1, 1))
self.inp[0] = 1
self.outp = np.ndarray((n_out, 1))
self.deltas = np.zeros((n_out, 1))
# You can have better initialization here
if weights is None:
self.weight = np.random.rand(n_in, n_out)/10
else:
self.weights = weights
self.is_classifier_layer = is_classifier_layer
# Some handy properties of the layers
self.size = self.n_out
self.shape = self.weights.shape
def __init__(self, train, valid, test, layers=None, input_weights=None,
output_task='classification', output_activation='softmax',
cost='crossentropy', learning_rate=0.01, epochs=50):
"""
A MNIST recognizer
Parameters
----------
train : list
valid : list
test : list
learning_rate : float
epochs : positive int
Attributes
----------
training_set : list
validation_set : list
test_set : list
learning_rate : float
epochs : positive int
performances: array of floats
"""
self.learning_rate = learning_rate
self.epochs = epochs
self.classification_task = True if output_task == 'classification' else False # Either classification or regression
self.output_activation = output_activation
self.output_activation_func = Activation.get_activation(self.output_activation)
self.cost = cost
if self.cost == 'crossentropy':
self.cost_function = CrossEntropyError()
else:
# nothing else supported...
raise ValueError('not supported')
self.training_set = train
self.validation_set = valid
self.test_set = test
# Record the performance of each epoch for later usages
# e.g. plotting, reporting..
self.performances = []
self.layers = layers
self.input_weights = input_weights
# add bias values ("1"s) at the beginning of all data sets
self.training_set.input = np.insert(self.training_set.input, 0, 1,
axis=1)
self.validation_set.input = np.insert(self.validation_set.input, 0, 1,
axis=1)
self.test_set.input = np.insert(self.test_set.input, 0, 1, axis=1)
def _train_one_epoch(self):
"""
Train one epoch, seeing all input instances
"""
for img in self.training_set.input:
self.noise = 0.1
noisy = img + self.noise * np.random.uniform(-1.0,1.0)
normalized = Activation.tanh(noisy)
self.MLP._feed_forward(normalized)
self.MLP._compute_error(normalized[1:])
self.MLP._update_weights()
pass
def __init__(self, train, valid, test,
learningRate=0.01, epochs=50,
activation='sigmoid',
error='mse'):
self.learningRate = learningRate
self.epochs = epochs
self.trainingSet = train
self.validationSet = valid
self.testSet = test
# Initialize the weight vector with small random values
# between -0.3 and 0.3 to encourage sigmoid function learning
self.weight = np.random.rand(self.trainingSet.input.shape[1]) * 0.6 - 0.3
# np.ones(self.trainingSet.input.shape[1])
self.activation = Activation.getActivation(activation)
self.activationPrime = Activation.getDerivative(activation)
self.activationString = activation[0].upper() + activation[1:]
self.erString = error
if error == 'absolute':
self.erf = erf.AbsoluteError()
elif error == 'different':
self.erf = erf.DifferentError()
elif error == 'mse':
self.erf = erf.MeanSquaredError()
elif error == 'sse':
self.erf = erf.SumSquaredError()
elif error == 'bce':
self.erf = erf.BinaryCrossEntropyError()
elif error == 'crossentropy':
self.erf = erf.CrossEntropyError()
else:
raise ValueError('Cannot instantiate the requested '
'error function: ' + error + 'not available')
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Write your code to do the classification on an input image
return Activation.sign(np.dot(testInstance, self.weight))
def train(self, verbose=True):
"""Train the perceptron with the perceptron learning algorithm.
Parameters
----------
verbose : boolean
Print logging messages with validation accuracy if verbose is True.
"""
# Write your code to train the perceptron here
pass
for epoch in range(self.epochs):
for i in range(self.trainingSet.input.shape[0]):
pre_result = dot(self.weight, self.trainingSet.input[i, :])
result = Activation.sign(pre_result, threshold=0)
error = self.trainingSet.label[i] - result
self.updateWeights(self.trainingSet.input[i, :], error)
correct = 0.0
for j in range(self.validationSet.input.shape[0]):
valid_result = Activation.sign(
dot(self.weight, self.validationSet.input[j, :]))
if valid_result == self.validationSet.label[j]:
correct += 1.0
accuracy = correct / self.validationSet.input.shape[0]
# print('after %d times training, valiation accuracy : %.4f %d' %(epoch, accuracy, correct))
# Den Schwellwert hab ich selbst auf 0.98 gesetzt
if accuracy >= 0.98:
if verbose:
print(
'After %d times training, Validation accuracy:%.4f>0.98'
% (epoch, accuracy))
print('Stop training to avoid overfitting!')
break
if epoch == self.epochs - 1:
print(
'No accuracy >= threshold, no need to break loop to avoid overfitting'
)
def forward(self, input):
"""
Compute forward step over the input using its weights
Parameters
----------
input : ndarray
a numpy array (1,nIn + 1) containing the input of the layer
Returns
-------
ndarray :
a numpy array (nOut,1) containing the output of the layer
"""
self.input[1:, :] = input.T
self.output = Activation.sigmoid(np.dot(self.weights, self.input))
return self.output
def classify(self, testInstance):
"""Classify a single instance.
Parameters
----------
testInstance : list of floats
Returns
-------
bool :
True if the testInstance is recognized as a 7, False otherwise.
"""
# Write your code to do the classification on an input image
pass
testResult = Activation.sign(dot(self.weight, testInstance),
threshold=0)
return testResult
def __init__(self, nIn, nOut, weights=None, activation='sigmoid'):
# Get activation function from string
# Notice the functional programming paradigms of Python + Numpy
self.activationString = activation
self.activation = Activation.getActivation(self.activationString)
self.nIn = nIn
self.nOut = nOut
# Some handy properties of the layers
self.size = self.nOut
self.shape = self.weights.shape
# You can have better initialization here
if weights is None:
rns = np.random.RandomState(int(time.time()))
self.weights = rns.uniform(size=(nOut, nIn + 1))
else:
self.weights = weights
def computeDerivative(self, nextDerivatives, nextWeights):
"""
Compute the derivatives (back)
Parameters
----------
nextDerivatives: ndarray
a numpy array containing the derivatives from next layer
nextWeights : ndarray
a numpy array containing the weights from next layer
Returns
-------
ndarray :
a numpy array containing the partial derivatives on this layer
"""
temp1 = np.dot(nextDerivatives.reshape(nextDerivatives.size, 1),
nextWeights.reshape(1, nextWeights.size))
temp2 = (np.sum(temp1, axis=0)).reshape(self.size, 1)
self.delta = (Activation.sigmoidPrime(self.output)) * temp2
pass
def __init__(self, n_in, n_out, weights=None,
activation='sigmoid', is_classifier_layer=False):
# Get activation function from string
self.activation_string = activation
self.activation = Activation.get_activation(self.activation_string)
self.n_in = n_in
self.n_out = n_out
#self.inp = np.ndarray((n_in+1, 1))
self.inp = np.ndarray((n_in+1))
self.inp[0] = 1
#self.outp = np.ndarray((n_out, 1))
self.outp = np.ndarray((n_out))
#self.deltas = np.zeros((n_out, 1))
self.deltas = np.zeros((n_out))
# You can have better initialization here
if weights is None:
self.weights = np.random.rand(n_in+1, n_out)/10
else:
self.weights = weights
def _fire(self, inp):
return Activation.sigmoid(np.dot(np.array(inp), self.weight))
def fire(self, input):
"""Fire the output of the perceptron corresponding to the input """
return Activation.sign(np.dot(np.array(input), self.weight))
def fire(self, input):
"""Fire the output of the perceptron corresponding to the input """
# I already implemented it for you to see how you can work with numpy
return Activation.sign(np.dot(np.array(input), self.weight[1:]) + self.weight[0])
def _fire(self, inp):
#print np.array(inp).shape
#print np.array(self.weights).shape
return Activation.sigmoid(np.dot(np.array(inp), self.weights))
pass
def _fire(self, inp, weightsOfNeuron):
return Activation.sigmoid(np.dot(np.array(inp), np.array(weightsOfNeuron)))
def _fire(self, inp):
return Activation.sigmoid(np.dot(np.array(inp), self.weights))
def fire(self, input):
return Activation.sigmoid(np.dot(np.array(input), self.weight))
def fire(self, input):
# input (n,1)
return Activation.sigmoid(np.dot(self.weight,input))
def fire(self, input):
"""Fire the output of the perceptron corresponding to the input """
return Activation.sign(np.dot(np.array(input), self.weight))
def _fire(self, inp):
#TODO compute a vector containing all sigmoids of neurons
ret = np.zeros(self.n_out)
for i in range(0, self.n_out):
ret[i] = Activation.sigmoid(np.dot(np.append(1,inp), self.weights[:,i]))
return ret
def _fire(self, input):
"""Fire the output of the perceptron corresponding to the input """
# I already implemented it for you to see how you can work with numpy
return Activation.sign(np.dot(np.array(input), self.weight))
def fire(self, input):
# Look at how we change the activation function here!!!!
# Not Activation.sign as in the perceptron, but sigmoid
return Activation.sigmoid(np.dot(np.array(input), self.weight))
