class Activation_Softmax_Loss_CategoricalCorssentropy():
# Create activation and loss function object
def __init__(self):
self.activation = Activation_Softmax()
self.loss = Loss_CategoricalCrossentropy()
def forward(self, inputs, y_true):
# Output layer's activation function
self.activation.forward(inputs)
# Set the output
self.output = self.activation.output
# Calculate and return loss value
return self.loss.calculate(self.output, y_true)
# Backward pass
def backward(self, dvalues, y_true):
# Number of samples
samples = len(dvalues)
# If labels are one-hot encoded,
# turn them into discrete values
if len(y_true.shape) == 2:
# argmax with axis 1, will return index the higest value
y_true = np.argmax(y_true, axis=1)
# Copy so we can safely modify
self.dinputs = dvalues.copy()
# Calculate gradient
self.dinputs[range(samples), y_true] -= 1
# Normalize gradient
self.dinputs = self.dinputs / samples
class Activation_Softmax_Loss_CategoricalCorssentropy():
# Create activation and loss function object
def __init__(self):
self.activation = Activation_Softmax()
self.loss = Loss_CategoricalCrossentropy()
def forward(self, inputs, y_true):
# Output layer's activation function
self.activation.forward(inputs)
# Set the output
self.output = self.activation.output
# Calculate and return loss value
return self.loss.calculate(self.output, y_true)
# Regularizatoin loss calculation
def regularization_loss(self, layer):
# 0 by default
regularization_loss = 0
# L1 regularization - weigths
# calculate only when factor greater than 0
if layer.weigth_regularizer_l1 > 0:
regularization_loss += layer.weigth_regularizer_l1 * np.sum(
np.abs(layer.weights))
# L2 regularization - weights
if layer.weight_regularizer_l2 > 0:
regularization_loss += layer.weight_regularizer_l2 * np.sum(
layer.weights * layer.weights)
# L1 regularization - biases
if layer.bias_regularizer_l1 > 0:
regularization_loss += layer.bias_regularizer_l1 * np.sum(
np.abs(layer.biases))
# L2 regularization - bises
if layer.bias_regularizer_l2 > 0:
regularization_loss += layer.bias_regularizer_l2 * np.sum(
layer.biases * layer.biases)
return regularization_loss
# Backward pass
def backward(self, dvalues, y_true):
# Number of samples
samples = len(dvalues)
# If labels are one-hot encoded,
# turn them into discrete values
if len(y_true.shape) == 2:
# argmax with axis 1, will return index the higest value
y_true = np.argmax(y_true, axis=1)
# Copy so we can safely modify
self.dinputs = dvalues.copy()
# Calculate gradient
self.dinputs[range(samples), y_true] -= 1
# Normalize gradient
self.dinputs = self.dinputs / samples
dense2.weights += 0.05 * np.random.randn(3, 3)
dense2.biases += 0.05 * np.random.randn(1, 3)
# Perform a forward pass of the training data through this layer
dense1.forward(x)
# we use Activation ReLU for hidden layer
activation_relu.forward(dense1.output)
dense2.forward(activation_relu.output)
# we use Activation Softmax for output layer
activation_softmax.forward(dense2.output)
# Perform a forward pass through activation function
# it takes the output of second dense layer here and returns loss
loss = loss_function.calculate(activation_softmax.output, y)
# Calculate accuracy from output of activation2 and tragets
# calculate values along first axis
predictions = np.argmax(activation_softmax.output, axis=1)
accuracy = np.mean(predictions == y)
# If loss is smaller - print and save weights and biases asid
if loss < lowest_loss:
print('New set of weight found, iteration:', iteration, 'loss:', loss,
'accucray', accuracy)
best_dense1_weights = dense1.weights.copy()
best_dense1_biases = dense1.biases.copy()
best_dense2_weights = dense2.weights.copy()
best_dense2_biases = dense2.biases.copy()
import math import numpy as np from libraries import Loss_CategoricalCrossentropy # Consider a scenario with a neural network that performs classification between three classes, and the neural network classifies in batches of threee. # After running through the softmax activation function with a batch of samples and 3 classes, the network's output layer yields: # Probabilities for 3 samples softmax_outputs = np.array([[0.7, 0, 0.2], [0.1, 0.5, 0.4], [0.02, 0.9, 0.08]]) # Let asumes we trying to classify something as a "dog", "cat" or "human". # A dos is class 0 (at index 0), cat class 1 (index 1), and a human class 2 (index 2) # in the book the author say it categorical labels # class_target = np.array([0, 1, 1]) class_targets = np.array([[1, 0, 0], [0, 1, 0], [0, 1, 0]]) loss_class = Loss_CategoricalCrossentropy() loss = loss_class.calculate(softmax_outputs, class_targets) print(loss)