class MnistNetMiniBatch:
def __init__(self):
self.d1_layer = Dense(784, 100)
self.a1_layer = ReLu()
self.drop1_layer = Dropout(0.5)
self.d2_layer = Dense(100, 50)
self.a2_layer = ReLu()
self.drop2_layer = Dropout(0.25)
self.d3_layer = Dense(50, 10)
self.a3_layer = Softmax()
def forward(self, x, train=True):
net = self.d1_layer.forward(x)
net = self.a1_layer.forward(net)
net = self.drop1_layer.forward(net, train)
net = self.d2_layer.forward(net)
net = self.a2_layer.forward(net)
net = self.drop2_layer.forward(net, train)
net = self.d3_layer.forward(net)
net = self.a3_layer.forward(net)
return (net)
def backward(self,
dz,
learning_rate=0.01,
mini_batch=True,
update=False,
len_mini_batch=None):
dz = self.a3_layer.backward(dz)
dz = self.d3_layer.backward(dz,
learning_rate=learning_rate,
mini_batch=mini_batch,
update=update,
len_mini_batch=len_mini_batch)
dz = self.drop2_layer.backward(dz)
dz = self.a2_layer.backward(dz)
dz = self.d2_layer.backward(dz,
learning_rate=learning_rate,
mini_batch=mini_batch,
update=update,
len_mini_batch=len_mini_batch)
dz = self.drop1_layer.backward(dz)
dz = self.a1_layer.backward(dz)
dz = self.d1_layer.backward(dz,
learning_rate=learning_rate,
mini_batch=mini_batch,
update=update,
len_mini_batch=len_mini_batch)
return dz
def test_dense_layer_NUMERICAL_GRADIENT_CHECK(self):
x = np.linspace(-1, 1, 10 * 32).reshape([10, 32])
l = Dense(32, 64)
numeric_grads = eval_numerical_gradient(lambda x: l.forward(x).sum(),
x)
grads = l.backward(x, np.ones([10, 64]), optim='gd', lr=0)
self.assertTrue(np.allclose(grads, numeric_grads, rtol=1e-5, atol=0),
msg="input gradient does not match numeric grad")
def test_dense_layer_FORWARD(self):
layer = Dense(3, 4)
x = np.linspace(-1, 1, 2 * 3).reshape([2, 3])
layer.weights = np.linspace(-1, 1, 3 * 4).reshape([3, 4])
layer.biases = np.linspace(-1, 1, 4)
self.assertTrue(
np.allclose(
layer.forward(x),
np.array([[0.07272727, 0.41212121, 0.75151515, 1.09090909],
[-0.90909091, 0.08484848, 1.07878788, 2.07272727]])))
layer1 = Dense(trainingData.shape[2], 16)
activation1 = Sigmoid()
layer2 = Dense(16, 10)
activation2 = SoftMax()
cost = CostMeanSquared()
for epoch in range(EPOCHS):
print('Epoch: ' + str(epoch + 1) + '/' + str(EPOCHS))
print('')
correct = 0
for batch in range(total // BATCH_SIZE):
### SOCHASIC GRADIENT DESCENT ###
layer1.forward(trainingData[batch])
activation1.forward(layer1.outputs)
layer2.forward(activation1.outputs)
activation2.forward(layer2.outputs)
cost.forward(activation2.outputs, labels[batch], 10)
for sample in range(activation2.outputs.shape[1]):
if np.argmax(activation2.outputs[:, sample]) == np.argmax(
labels[batch, sample]):
correct += 1
cost.backward(activation2.outputs, labels[batch], 10)
activation2.backward(layer2.outputs, layer2.weights.shape[0],
BATCH_SIZE)
layer2.backward(activation1.outputs)
activation1.backward(layer1.outputs)
W2 = np.array([[0.4, 0.45], [0.50, 0.55]])
b2 = 0.60
y_true = np.array([[0.01, 0.99]])
dense = Dense(2, W1, b1)
sigmoid = Activations('Sigmoid')
swish1 = Activations('Swish')
dense2 = Dense(2, W2, b2)
swish2 = Activations('Swish')
activation2 = Activations('Sigmoid')
loss_fn = MSE()
z1 = dense.forward(x)
sig1 = sigmoid.forward(z1)
z2 = dense2.forward(sig1)
y_pred = activation2.forward(z2)
sw1 = swish1.forward(z1)
sw2 = dense2.forward(sw1)
y_pre = swish2.forward(sw2)
# loss = loss_fn.loss(y_true, y_pred)
# print("loss: ", loss)
# print("loss's mean: ",np.mean(loss))
# sigloss = loss_fn.loss(y_true, y_pre)
# print("SIG loss: ", sigloss)
# print("sig loss's mean: ",np.mean(sigloss))
def compute_out_given_wb(w, b):
layer = Dense(32, 64)
layer.weights = np.array(w)
layer.biases = np.array(b)
x = np.linspace(-1, 1, 10 * 32).reshape([10, 32])
return layer.forward(x)
y_true = np.array([[.01, .99]])
#Layers Generation
dense = Dense(2, W1, b1)
dense2 = Dense(2, W2, b2)
activation1 = Sigmoid()
# activation2=Sigmoid()
activation2 = Activation("sigmoid")
loss_func = MSE()
#Forward Pass
# Dense -> Activation -> Dense -> Activation -> y_pred
z1 = dense.forward(x)
a1 = activation1.forward(z1)
print("Activation Value:", a1)
z2 = dense2.forward(a1)
a2 = activation2.forward(z2)
y_pred = a2
loss = loss_func.loss(y_true, y_pred)
print("Individual Loss:", loss)
total_loss = np.mean(loss)
print("Total Loss:", total_loss)
#Backward Propagation
dLdy_pred = loss_func.gradient(y_true, y_pred)
