def test_delta_cross_entropy_loss(self):
# settings
batch_size = 4
N_out = 10
precision = 0.000001
# tensors
outputs = torch.rand((batch_size, N_out),
dtype=torch.float,
requires_grad=True)
labels = torch.randint(high=10, size=(batch_size, ), dtype=torch.long)
outputs_s = outputs
grads_creloss = loss.delta_cross_entropy_softmax(
activation.softmax(outputs), labels)
closs = nn.CrossEntropyLoss()
xloss = closs(outputs, labels)
xloss.backward()
grads_creloss = loss.delta_cross_entropy_softmax(outputs, labels)
assert isinstance(grads_creloss, torch.FloatTensor)
assert grads_creloss.size() == torch.Size([batch_size, N_out])
self.assertTrue(
torch.le(torch.abs(grads_creloss - outputs.grad), precision).all())
def test_delta_cross_entropy_loss(self):
# settings
batch_size = 11
N_out = 17
precision = 0.000001
# tensors
outputs = torch.rand((batch_size, N_out),
dtype=torch.float,
requires_grad=True)
labels = torch.randint(high=N_out,
size=(batch_size, ),
dtype=torch.long)
# calculate gradients from scratch
grads_creloss = loss.delta_cross_entropy_softmax(
activation.softmax(outputs), labels)
# calculate gradients with autograd
nll = torch.nn.functional.cross_entropy(outputs, labels)
nll.backward()
assert isinstance(grads_creloss, torch.FloatTensor)
assert grads_creloss.size() == torch.Size([batch_size, N_out])
self.assertTrue(
torch.le(torch.abs(grads_creloss - outputs.grad), precision).all())
def backward(self, inputs, labels, outputs):
"""Backward pass of neural network
Changes weights and biases of each layer to reduce loss
Args:
inputs (torch.tensor): inputs to train neural network. Size (batch_size, N_in)
labels (torch.tensor): correct labels. Size (batch_size)
outputs (torch.tensor): outputs predicted by neural network. Size (batch_size, N_out)
Returns:
dw1 (torch.tensor): Gradient of loss w.r.t. w1. Size like w1
db1 (torch.tensor): Gradient of loss w.r.t. b1. Size like b1
dw2 (torch.tensor): Gradient of loss w.r.t. w2. Size like w2
db2 (torch.tensor): Gradient of loss w.r.t. b2. Size like b2
dw3 (torch.tensor): Gradient of loss w.r.t. w3. Size like w3
db3 (torch.tensor): Gradient of loss w.r.t. b3. Size like b3
"""
# Calculating derivative of loss w.r.t weighted sum
dout = loss.delta_cross_entropy_softmax(outputs, labels)
# d2 = (dout.w3)*(a2)
d2 = torch.mul(torch.matmul(dout, self.weights['w3']),
activation.delta_sigmoid(self.cache['z2']))
# d1 = (d2.w2)*(a1)
d1 = torch.mul(torch.matmul(d2, self.weights['w2']),
activation.delta_sigmoid(self.cache['z1']))
dw1, db1, dw2, db2, dw3, db3 = self.calculate_grad(
inputs, d1, d2, dout) # calculate all gradients
return dw1, db1, dw2, db2, dw3, db3
def test_delta_cross_entropy_loss(self):
# settings
batch_size = 4
N_out = 10
# tensors
outputs = torch.rand((batch_size, N_out), dtype=torch.float)
labels = torch.randint(high=10, size=(batch_size, ), dtype=torch.long)
grads_creloss = loss.delta_cross_entropy_softmax(outputs, labels)
assert isinstance(grads_creloss, torch.FloatTensor)
assert grads_creloss.size() == torch.Size([batch_size, N_out])
# write more robust test cases here
# you should write gradient checking code here
pass