def test_sigmoid_backward():
# Test input
np.random.seed(0)
autograd = autograd_engine.Autograd()
l1 = nn.Linear(5, 5, autograd)
x = np.random.random((1, 5))
l1_out = l1(x)
test_act = nn.Sigmoid(autograd)
a1_out = test_act(l1_out)
autograd.backward(1)
# Torch input
torch_l1 = torch.nn.Linear(5, 5)
torch_l1.weight = torch.nn.Parameter(torch.DoubleTensor(l1.W))
torch_l1.bias = torch.nn.Parameter(torch.DoubleTensor(l1.b.squeeze()))
torch_x = torch.DoubleTensor(x)
torch_l1_out = torch_l1(torch_x)
torch_act = torch.nn.Sigmoid()
torch_a1_out = torch_act(torch_l1_out)
torch_a1_out.sum().backward()
compare_np_torch(l1.dW, torch_l1.weight.grad)
compare_np_torch(l1.db.squeeze(), torch_l1.bias.grad)
return True
def test_softmaxXentropy_forward():
# Test input
np.random.seed(0)
autograd = autograd_engine.Autograd()
l1 = nn.Linear(5, 5, autograd)
x = np.random.random((1, 5))
y = np.array([[0., 0., 1., 0., 0.]])
l1_out = l1(x)
test_loss = nn.SoftmaxCrossEntropy(autograd)
a1_out = test_loss(y, l1_out)
# Torch input
torch_l1 = torch.nn.Linear(5, 5)
torch_l1.weight = torch.nn.Parameter(torch.DoubleTensor(l1.W))
torch_l1.bias = torch.nn.Parameter(torch.DoubleTensor(l1.b.squeeze()))
torch_x = torch.DoubleTensor(x)
torch_y = torch.LongTensor(np.array([2]))
torch_l1_out = torch_l1(torch_x)
torch_loss = torch.nn.CrossEntropyLoss()
torch_a1_out = torch_loss(torch_l1_out, torch_y).reshape(1, )
torch_a1_out.backward()
compare_np_torch(a1_out, torch_a1_out)
return True
def test_identity_forward():
# Test input
np.random.seed(0)
autograd = autograd_engine.Autograd()
l1 = nn.Linear(5, 5, autograd)
x = np.random.random(
(1, 5)) # just use batch size 1 since broadcasting is not written yet
l1_out = l1(x)
test_act = nn.Identity(autograd)
a1_out = test_act(l1_out)
# Torch input
torch_l1 = torch.nn.Linear(5, 5)
torch_l1.weight = torch.nn.Parameter(torch.DoubleTensor(
l1.W)) # note transpose here, probably should standardize
torch_l1.bias = torch.nn.Parameter(torch.DoubleTensor(l1.b.squeeze()))
torch_x = torch.DoubleTensor(x)
torch_l1_out = torch_l1(torch_x)
torch_act = torch.nn.Identity()
torch_a1_out = torch_act(torch_l1_out)
compare_np_torch(a1_out, torch_a1_out)
return True
def test_linear_skip_backward():
np.random.seed(0)
autograd = autograd_engine.Autograd()
autograd.zero_grad()
l1 = nn.Linear(5, 5, autograd)
x = np.random.random((1, 5))
l1_out = l1(x)
output = l1_out + x
autograd.add_operation(inputs=[l1_out, x], output=output, gradients_to_update=[None, None],
backward_operation=add_backward)
autograd.backward(1)
torch_l1 = torch.nn.Linear(5, 5)
torch_l1.weight = torch.nn.Parameter(torch.DoubleTensor(l1.W))
torch_l1.bias = torch.nn.Parameter(torch.DoubleTensor(l1.b.squeeze()))
torch_x = torch.DoubleTensor(x)
torch_x.requires_grad = True
torch_l1_out = torch_l1(torch_x)
torch_output = torch_l1_out + torch_x
torch_output.sum().backward()
compare_np_torch(l1_out, torch_l1_out)
compare_np_torch(l1.dW, torch_l1.weight.grad)
compare_np_torch(l1.db.squeeze(), torch_l1.bias.grad)
compare_np_torch(autograd.memory_buffer.get_param(x), torch_x.grad) # skip connections work'''
return True
def test_linear_skip_forward():
np.random.seed(0)
autograd = autograd_engine.Autograd()
autograd.zero_grad()
l1 = nn.Linear(5, 5, autograd)
x = np.random.random((1, 5))
l1_out = l1(x)
output = l1_out + x
autograd.add_operation(inputs=[l1_out, x], output=output, gradients_to_update=[None, None],
backward_operation=add_backward)
torch_l1 = torch.nn.Linear(5, 5)
torch_l1.weight = torch.nn.Parameter(torch.DoubleTensor(l1.W))
torch_l1.bias = torch.nn.Parameter(torch.DoubleTensor(l1.b.squeeze()))
torch_x = torch.DoubleTensor(x)
torch_x.requires_grad = True
torch_l1_out = torch_l1(torch_x)
torch_output = torch_l1_out + torch_x
compare_np_torch(output, torch_output)
return True
def test_linear_layer_forward():
np.random.seed(0)
x = np.random.random((1, 5))
autograd = autograd_engine.Autograd()
l1 = nn.Linear(5, 5, autograd)
l1_out = l1(x)
torch_l1 = torch.nn.Linear(5, 5)
torch_l1.weight = torch.nn.Parameter(torch.DoubleTensor(l1.W))
torch_l1.bias = torch.nn.Parameter(torch.DoubleTensor(l1.b.squeeze()))
torch_x = torch.DoubleTensor(x)
torch_l1_out = torch_l1(torch_x)
compare_np_torch(l1_out, torch_l1_out)
return True
def test_tanh_forward():
# Test input
np.random.seed(0)
autograd = autograd_engine.Autograd()
l1 = nn.Linear(5, 5, autograd)
x = np.random.random((1, 5))
l1_out = l1(x)
test_act = nn.Tanh(autograd)
a1_out = test_act(l1_out)
# Torch input
torch_l1 = torch.nn.Linear(5, 5)
torch_l1.weight = torch.nn.Parameter(torch.DoubleTensor(l1.W))
torch_l1.bias = torch.nn.Parameter(torch.DoubleTensor(l1.b.squeeze()))
torch_x = torch.DoubleTensor(x)
torch_l1_out = torch_l1(torch_x)
torch_act = torch.nn.Tanh()
torch_a1_out = torch_act(torch_l1_out)
compare_np_torch(a1_out, torch_a1_out)
return True
