def conv_layer(x):
"""
the derivative check in the gradient checker relates to the input of the function
hence, the input should be z - since the backward step computes @loss / @z
"""
conv1 = nn.Conv2d(in_channels=1, out_channels=2, kernel_size=2)
sigmoid1 = nn.Sigmoid()
conv2 = nn.Conv2d(in_channels=2, out_channels=4, kernel_size=2)
sigmoid2 = nn.Sigmoid()
flatten = nn.Flatten()
linear = nn.Linear(4, 2)
softmax = nn.Softmax()
# forward pass
a = sigmoid1(conv1(x))
a = sigmoid2(conv2(a))
a_flatten = flatten(a)
dist = softmax(linear(a_flatten))
# backward
labels = np.zeros(dist.shape)
labels[:, 1] = 1
loss = -np.log(np.sum(dist * labels, axis=1))
softmax_grad = softmax.backward(labels)
linear_grad = linear.backward(softmax_grad)
flatten_grad = flatten.backward(linear_grad)
sigmoid2_grad = sigmoid2.backward(flatten_grad)
conv2_grad = conv2.backward(sigmoid2_grad)
sigmoid1_grad = sigmoid1.backward(conv2_grad)
conv1_grad = conv1.backward(sigmoid1_grad)
return loss, conv1_grad
def test_forwardWrongInputListDimension(self):
m = M.Sigmoid()
try:
m.forward(FloatTensor([[2]]), FloatTensor([[2]]))
except ValueError:
return 0
return 1
def test_backwardBeforeForward(self):
input = FloatTensor([[1, -3, 1], [0, -1, 2]])
m = M.Sigmoid()
try:
output = m.backward(input)
except:
return 0
return 1
def test_forwardCorrectOutput(self):
input = FloatTensor([[1, -3, 1], [0, -1, 2]])
expected_output = FloatTensor([[0.731058578630005,0.047425873177567,0.731058578630005], \
[0.500000000000000,0.268941421369995,0.880797077977882]])
m = M.Sigmoid()
output = m.forward(input)
if not areEqual(output, expected_output, tol=1e-5):
return 1
return 0
def sigmoid_layer(x):
sigmoid = nn.Sigmoid()
softmax = nn.Softmax()
a = softmax(sigmoid(x))
num_classes = x.shape
labels = np.zeros(num_classes)
labels[:, 0] = 1
loss = -np.log(np.sum(a * labels, axis=1))
softmax_grad = softmax.backward(labels)
sigmoid_grad = sigmoid.backward(softmax_grad)
return loss, sigmoid_grad
def test_backwardCorrectOutput(self):
input = FloatTensor([[1, -3, 1], [0, -1, 2]])
sigmoid_value = FloatTensor([[0.731058578630005,0.047425873177567,0.731058578630005], \
[0.500000000000000,0.268941421369995,0.880797077977882]])
dsigma = sigmoid_value * (1 - sigmoid_value)
m = M.Sigmoid()
m.forward(input)
grad = FloatTensor([[1, 2, 3], [3, 2, 1]])
output = m.backward(grad)
output_expected = dsigma * grad
if not areEqual(output, output_expected):
return 1
return 0