def test_back_tanh():
da = torch.randn(3, 4) # 3 nodes in layer, 4 examples
z = torch.randn(3, 4) # 3 nodes in layer, 4 examples
dz = back_tanh(da, z)
a = tanh(z)
expected_value = da[1, 2] * (1 - a * a)[1, 2]
assert math.isclose(dz[1, 2], expected_value, rel_tol=1e-05)
def forward_activate(z, activationFunctionID):
if activationFunctionID == 'sigmoid':
return sigmoid(z)
elif activationFunctionID == 'tanh':
return tanh(z)
elif activationFunctionID == 'softmax':
return softmax(z)
else:
assert(False) # Unrecognized activation function ID string
def test_tanh_back():
zr = torch.randn(3,4)
zr_copy = zr.clone().detach()
zr_copy = tanh(zr_copy)
ar = tanh_back(zr)
zr_shape = zr.shape
# Check that each element is 1 - tanh(z) * tanh(z),
# where z is an element in the zr matrix.
for i in range(0, zr_shape[0]):
for j in range(0, zr_shape[1]):
expected_value = 1.0 - (zr_copy[i][j] * zr_copy[i][j])
assert math.isclose(ar[i][j], expected_value, rel_tol=1e-05)
def test_tanh():
zr = torch.randn(3,4)
zr_copy = zr.clone().detach()
ar = tanh(zr)
zr_shape = zr.shape
# Check that each element in ar is equal to (e**x - e**-x)/(e**x + e**-x)
# tanh is implemented as (2 / (1 + e**-2z)) - 1 in dl_activate
for i in range(0, zr_shape[0]):
for j in range(0, zr_shape[1]):
ex = math.exp(zr_copy[i][j]) # e**x
emx = math.exp(-zr_copy[i][j]) # e**-x
expected_value = (ex - emx)/(ex + emx)
assert math.isclose(ar[i][j], expected_value, rel_tol=1e-04)
def test_mlp_train_2Layer_softmax_costs():
device = get_device()
X = torch.randn(5, 10).to(device)
y = torch.randn(3, 10).to(device)
activationFunctionID0 = 'tanh'
activationFunctionID1 = 'softmax'
layer0 = NNLayerConfig(4, activationFunctionID0)
layer1 = NNLayerConfig(3, activationFunctionID1)
layers = [layer0, layer1]
numInputNodes = X.shape[0]
mlp = MLPModel(numInputNodes, layers)
weightsCopy = []
for weight in mlp.m_weights:
weightsCopy.append(weight.clone().detach())
biasesCopy = []
for bias in mlp.m_biases:
biasesCopy.append(bias.clone().detach())
lossFunctionID = 'loss_cross_entropy_softmax'
numEpochs = 1
batchSize = 4
optimizer = AdamOptimizer(mlp)
regularizationFactor = 0
regularizer = L2Regularizer(regularizationFactor)
numBatches, costs = mlp_train(mlp, X, y, lossFunctionID, regularizer,
optimizer, batchSize, numEpochs)
assert math.isclose(len(costs), 3, rel_tol=1e-05)
# Check first batch
xBatch0 = X[:, 0:4]
yBatch0 = y[:, 0:4]
assert math.isclose(xBatch0.shape[1], 4, rel_tol=1e-05)
z0 = torch.matmul(weightsCopy[0], xBatch0) + biasesCopy[0]
a0 = tanh(z0)
z1 = torch.matmul(weightsCopy[1], a0) + biasesCopy[1]
a1 = softmax(z1)
y_pred = a1
loss0 = compute_loss(yBatch0, y_pred, lossFunctionID)
cost0 = torch.true_divide(torch.sum(loss0, dim=1), loss0.shape[1])
cost0 = cost0.to(device)
assert torch.allclose(costs[0], cost0)
assert costs[0].shape[0] == 3
def test_forward_rnn_tanh():
xr = torch.randn(4, 3)
aPrevr = torch.randn(2, 3)
war = torch.randn(3, 6)
bar = torch.randn(3, 1)
activationFunctionID = 'tanh'
z, a = forward_rnn(xr, aPrevr, war, bar, activationFunctionID)
xa = torch.cat((aPrevr, xr), 0)
assert xa.shape[0] == 6
assert xa.shape[1] == 3
zExpected = torch.matmul(war, xa) + bar
assert torch.allclose(z, zExpected)
aExpected = tanh(zExpected)
assert torch.allclose(a, aExpected)
def test_forward_tanh():
# x is a 4x3 matrix. 3 examples, 4 features or nodes per example.
xr = torch.randn(4, 3)
# w is 2x4 matrix. 2 nodes in the layer being forward propagated. 4 nodes in the previous layer.
wr = torch.randn(2, 4)
# b is a 2x1 matrix, 2 constants for the 2 nodes in the layer being forward propagated.
br = torch.randn(2, 1)
# a, the forward propagation result in the layer is a 2x3 matrix. 2 nodes or features for each of the 3 examples.
activationFunctionID = 'tanh'
zr, ar = forward(xr, wr, br, activationFunctionID)
zExpected = torch.matmul(wr, xr) + br
aExpected = tanh(zExpected)
assert torch.allclose(zr, zExpected)
assert torch.allclose(ar, aExpected)