def activation_forward(self,input,W,b,activation_type): ''' :param input: the input of the current layer :param W: the weights of the current layer :param b: biases of the current layer :param activation_type: Type of activation function used in the forward propagation :return: - A --> the output of the activation function - packet_of_packets --> Tuple of 2 elements which will be used in backward propagation : 1- linear packer : contains ( input , weights , biases ) of the current layer 2- activation packet : contains ( Z ) which is the input to the activation function ''' if activation_type == "sigmoid": Z, linear_packet = self.identity_forward(input, W, b) ## Z = input * w + b temp=activations.Sigmoid() A, activation_packet = temp.forward(Z) ## A = sig(z) elif activation_type == "relu": Z, linear_packet = self.identity_forward(input, W, b) temp = activations.relu() A, activation_packet = temp.forward(Z) elif activation_type == "leaky_relu": Z, linear_packet = self.identity_forward(input, W, b) temp = activations.leaky_relu() A, activation_packet = temp.forward(Z) elif activation_type == "tanh": Z, linear_packet = self.identity_forward(input, W, b) temp = activations.tanh() A, activation_packet = temp.forward(Z) elif activation_type == "softmax": Z, linear_packet = self.identity_forward(input, W, b) #temp = A, activation_packet = activations.Softmax().forward(Z) elif activation_type == "linear": Z, linear_packet = self.identity_forward(input, W, b) # temp = A, activation_packet = Z,Z else: raise ValueError("ERROR : Activation Function is Not Determined") packet_of_packets = linear_packet, activation_packet return A, packet_of_packets
def __init__(self, f=''): # Forward computations self.x = None # layer input signal self.z = None # weighted sum (forward) self.a = None # activation (forward) self.f = None # Set activation if f == 'sigmoid': self.f = activations.Sigmoid() elif f == 'tanh': self.f = activations.Tanh() elif f == 'softmax': self.f = activations.Softmax() else: raise Exception('Currently need to specify activation.') # Back computations self.delta = None # delta for this layer
def activation_backward(self, delta_A, packet_of_packets, activation_type, lambd): ''' :param delta_A: the derivative of the loss function w.r.t the activation function :param packet_of_packets: Tuple of 2 elements which will be used in backward propagation : 1- linear packer : contains ( input , weights , biases ) of the current layer 2- activation packet : contains ( Z ) which is the input to the activation function :param activation_type: the type of the activation function used in this layer :param lambd: regularization parameter :return: - delta_input_previous , the gradient of the past input - delta_w : gradient of the weights of the current layer - delta_b : the gradient of the biases of the current layer ''' linear_packet, act_packet = packet_of_packets if activation_type == "relu": #print("hi") temp = activations.relu() dZ = temp.backaward( delta_A, act_packet) # we have to implement this relu backward function dA_prev, dW, db = self.identity_backward(dZ, linear_packet, lambd) elif activation_type == "sigmoid": #print("hi") temp = activations.Sigmoid() dZ = temp.backward(delta_A, act_packet) dA_prev, dW, db = self.identity_backward(dZ, linear_packet, lambd) # we will start from here tomorrow , we have to deal with Y_hat , y_true while creating instance from cost class # temp = Losses.square_difference() #dA = temp.backprop_cost(self.linear_packet) elif activation_type == "softmax": temp = activations.Softmax() dZ = temp.diff(delta_A) dA_prev, dW, db = self.identity_backward(dZ, linear_packet, lambd) return dA_prev, dW, db
def __init__(self): self.activation = activations.Softmax() self.loss = CategoricalCrossentropy() self._d_inputs = None
plt.plot(X_test, y_test) plt.plot(X_test, predictions) plt.show() ''' '' EPOCHS = 10001 LEARNING_RATE = 0.05 X_train, y_train = spiral_data(samples=100, classes=3) X_val, y_val = spiral_data(samples=100, classes=3) model = network.NeuralNetwork() model.add_layer( layers.Dense(2, 64, weight_regularizer_l2=0.000005, bias_regularizer_l2=0.000005)) model.add_layer(activations.ReLU()) model.add_layer(layers.Dropout(rate=0.2)) model.add_layer(layers.Dense(64, 3)) model.add_layer(activations.Softmax()) model.set(loss=losses.CategoricalCrossentropy(), optimizier=optimizers.Adam(learning_rate=LEARNING_RATE), accuracy=metrics.CategoricalAccuracy()) model.fit(X_train, y_train, epochs=EPOCHS, validation_data=(X_val, y_val))