def test_delta_cross_entropy_loss(self):
# settings
batch_size = 11
N_out = 17
precision = 0.000001
# tensors
outputs = torch.rand((batch_size, N_out),
dtype=torch.float,
requires_grad=True)
labels = torch.randint(high=N_out,
size=(batch_size, ),
dtype=torch.long)
# calculate gradients from scratch
grads_creloss = loss.delta_cross_entropy_softmax(
activation.softmax(outputs), labels)
# calculate gradients with autograd
nll = torch.nn.functional.cross_entropy(outputs, labels)
nll.backward()
assert isinstance(grads_creloss, torch.FloatTensor)
assert grads_creloss.size() == torch.Size([batch_size, N_out])
self.assertTrue(
torch.le(torch.abs(grads_creloss - outputs.grad), precision).all())
def test_delta_cross_entropy_loss(self):
# settings
batch_size = 4
N_out = 10
precision = 0.000001
# tensors
outputs = torch.rand((batch_size, N_out),
dtype=torch.float,
requires_grad=True)
labels = torch.randint(high=10, size=(batch_size, ), dtype=torch.long)
outputs_s = outputs
grads_creloss = loss.delta_cross_entropy_softmax(
activation.softmax(outputs), labels)
closs = nn.CrossEntropyLoss()
xloss = closs(outputs, labels)
xloss.backward()
grads_creloss = loss.delta_cross_entropy_softmax(outputs, labels)
assert isinstance(grads_creloss, torch.FloatTensor)
assert grads_creloss.size() == torch.Size([batch_size, N_out])
self.assertTrue(
torch.le(torch.abs(grads_creloss - outputs.grad), precision).all())
def delta_cross_entropy_softmax(outputs, labels):
"""Calculates derivative of cross entropy loss (C) w.r.t. weighted sum of inputs (Z).
"""
labels = labels.unsqueeze(0).float()
avg_grads = activation.softmax(outputs) - torch.t(labels)
return avg_grads
def test_softmax(self):
batch_size = 7
N_out = 10
x = torch.rand((batch_size, N_out), dtype=torch.float)
outputs = activation.softmax(x)
assert isinstance(outputs, torch.FloatTensor)
assert outputs.size() == torch.Size([batch_size, N_out])
def cross_entropy_loss(outputs, labels):
"""Calculates cross entropy loss given outputs and actual labels
"""
m = labels.shape[0]
p = activation.softmax(outputs)
log_likelihood = -torch.log(p[range(m), labels])
creloss = torch.sum(log_likelihood) / m
return creloss.item() # should return float not tensor
def delta_cross_entropy_softmax(outputs, labels):
"""Calculates derivative of cross entropy loss (C) w.r.t. weighted sum of inputs (Z).
"""
m = labels.shape[0]
grad = activation.softmax(outputs)
grad[range(m), labels] -= 1
avg_grads = grad / m
return avg_grads
def test_softmax(self):
batch_size = 7
N_out = 10
precision = 0.000001
x = torch.rand((batch_size, N_out), dtype=torch.float)
outputs = activation.softmax(x)
assert isinstance(outputs, torch.FloatTensor)
assert outputs.size() == torch.Size([batch_size, N_out])
self.assertTrue(torch.le(torch.abs(outputs - x.softmax(1)), precision).all())
def cross_entropy_loss(outputs, labels):
"""Calculates cross entropy loss given outputs and actual labels
"""
m, no_labels = outputs.size()
one_hot = torch.zeros(no_labels, m)
one_hot[labels, torch.arange(labels.size(0))] = 1
p = activation.softmax(outputs)
log_likelihood = -(torch.mul(torch.t(one_hot), torch.log(p)))
creloss = (1 / m) * torch.sum(log_likelihood)
return creloss.item() # should return float not tensor
def test_cross_entropy(self):
# settings
batch_size = 11
N_out = 12
# tensors
outputs = torch.rand((batch_size, N_out), dtype=torch.float)
labels = torch.randint(high=N_out,
size=(batch_size, ),
dtype=torch.long)
creloss = loss.cross_entropy_loss(activation.softmax(outputs), labels)
assert type(creloss) == float
# write more robust and rigourous test cases here
nll = torch.nn.functional.cross_entropy(outputs, labels)
self.assertAlmostEqual(creloss, nll.item(), places=6)
def test_cross_entropy(self):
# settings
batch_size = 4
N_out = 10
# tensors
outputs = torch.rand((batch_size, N_out), dtype=torch.float)
outputs_s = activation.softmax(outputs)
labels = torch.randint(high=10, size=(batch_size, ), dtype=torch.long)
closs = nn.CrossEntropyLoss()
creloss = loss.cross_entropy_loss(outputs_s, labels)
assert type(creloss) == float
# write more robust and rigourous test cases here
self.assertTrue(
math.isclose(creloss,
closs(outputs, labels).item(),
rel_tol=0.0009))
def test_softmax(self):
batch_size = 2
N_out = 8
x = torch.FloatTensor([[-10, -0.2, -0.6, 0, 0.1, 0.5, 2, 50],
[-10, -0.2, -0.6, 0, 0.1, 0.5, 2, 50]])
#already applied softmax values
y = torch.FloatTensor([[
8.7565e-27, 1.57913e-22, 1.05852e-22, 1.92875e-22, 2.1316e-22,
3.17997e-22, 1.42516e-21, 1
],
[
8.7565e-27, 1.57913e-22, 1.05852e-22,
1.92875e-22, 2.1316e-22, 3.17997e-22,
1.42516e-21, 1
]])
outputs = activation.softmax(x)
assert isinstance(outputs, torch.FloatTensor)
assert outputs.size() == torch.Size([batch_size, N_out])
precision = 0.0009
self.assertTrue(torch.le(torch.abs(outputs - y), precision).all())
def forward(self, inputs):
"""Forward pass of neural network
Calculates score for each class.
Args:
inputs (torch.tensor): inputs to train neural network. Size (batch_size, N_in)
Returns:
outputs (torch.tensor): predictions from neural network. Size (batch_size, N_out)
"""
#calculaing weighted sum & storing it in cache
self.cache['z1'] = self.weighted_sum(inputs, self.weights['w1'],
self.biases['b1'])
a1 = activation.sigmoid(self.cache['z1'])
self.cache['z2'] = self.weighted_sum(a1, self.weights['w2'],
self.biases['b2'])
a2 = activation.sigmoid(self.cache['z2'])
self.cache['z3'] = self.weighted_sum(a2, self.weights['w3'],
self.biases['b3'])
outputs = activation.softmax(self.cache['z3'])
return outputs