def forward_activate(z, activationFunctionID):
if activationFunctionID == 'sigmoid':
return sigmoid(z)
elif activationFunctionID == 'tanh':
return tanh(z)
elif activationFunctionID == 'softmax':
return softmax(z)
else:
assert(False) # Unrecognized activation function ID string
def test_softmax():
zr = torch.randn(3,4)
zr_copy = zr.clone().detach()
ar = softmax(zr)
zr_max, zr_max_idx = torch.max(zr, dim=0) # Get the max of each column
zr_copy = zr_copy - zr_max # Subtract the max of each column from all rows
zr_copy = torch.exp(zr_copy) # Take the exponent of all values
zr_copy_sum_cols = torch.sum(zr_copy, dim=0) # The the sum of each column
expected_value = zr_copy / zr_copy_sum_cols # Divide each row by the sum of each column
assert torch.allclose(ar, expected_value) # Check that ar is the same as the expected value.
def test_mlp_train_2Layer_softmax_costs():
device = get_device()
X = torch.randn(5, 10).to(device)
y = torch.randn(3, 10).to(device)
activationFunctionID0 = 'tanh'
activationFunctionID1 = 'softmax'
layer0 = NNLayerConfig(4, activationFunctionID0)
layer1 = NNLayerConfig(3, activationFunctionID1)
layers = [layer0, layer1]
numInputNodes = X.shape[0]
mlp = MLPModel(numInputNodes, layers)
weightsCopy = []
for weight in mlp.m_weights:
weightsCopy.append(weight.clone().detach())
biasesCopy = []
for bias in mlp.m_biases:
biasesCopy.append(bias.clone().detach())
lossFunctionID = 'loss_cross_entropy_softmax'
numEpochs = 1
batchSize = 4
optimizer = AdamOptimizer(mlp)
regularizationFactor = 0
regularizer = L2Regularizer(regularizationFactor)
numBatches, costs = mlp_train(mlp, X, y, lossFunctionID, regularizer,
optimizer, batchSize, numEpochs)
assert math.isclose(len(costs), 3, rel_tol=1e-05)
# Check first batch
xBatch0 = X[:, 0:4]
yBatch0 = y[:, 0:4]
assert math.isclose(xBatch0.shape[1], 4, rel_tol=1e-05)
z0 = torch.matmul(weightsCopy[0], xBatch0) + biasesCopy[0]
a0 = tanh(z0)
z1 = torch.matmul(weightsCopy[1], a0) + biasesCopy[1]
a1 = softmax(z1)
y_pred = a1
loss0 = compute_loss(yBatch0, y_pred, lossFunctionID)
cost0 = torch.true_divide(torch.sum(loss0, dim=1), loss0.shape[1])
cost0 = cost0.to(device)
assert torch.allclose(costs[0], cost0)
assert costs[0].shape[0] == 3
def test_forward_rnn_softmax():
xr = torch.randn(4, 3)
aPrevr = torch.randn(2, 3)
war = torch.randn(3, 6)
bar = torch.randn(3, 1)
activationFunctionID = 'softmax'
z, a = forward_rnn(xr, aPrevr, war, bar, activationFunctionID)
xa = torch.cat((aPrevr, xr), 0)
assert xa.shape[0] == 6
assert xa.shape[1] == 3
zExpected = torch.matmul(war, xa) + bar
assert torch.allclose(z, zExpected)
aExpected = softmax(zExpected)
assert torch.allclose(a, aExpected)
def test_forward_softmax():
# x is a 4x3 matrix. 3 examples, 4 features or nodes per example.
xr = torch.randn(4, 3)
# w is 2x4 matrix. 2 nodes in the layer being forward propagated. 4 nodes in the previous layer.
wr = torch.randn(2, 4)
# b is a 2x1 matrix, 2 constants for the 2 nodes in the layer being forward propagated.
br = torch.randn(2, 1)
# a, the forward propagation result in the layer is a 2x3 matrix. 2 nodes or features for each of the 3 examples.
activationFunctionID = 'softmax'
zr, ar = forward(xr, wr, br, activationFunctionID)
zExpected = torch.matmul(wr, xr) + br
aExpected = softmax(zExpected)
assert torch.allclose(zr, zExpected)
assert torch.allclose(ar, aExpected)