def test_momentum():
data = saved_data[21]
assert len(data) == 8
x = data[0]
y = data[1]
solW = data[2:5]
solb = data[5:]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [hw1.Sigmoid(), hw1.Sigmoid(), hw1.Identity()], weight_init, bias_init,
hw1.SoftmaxCrossEntropy(), 0.008,
momentum=0.856, num_bn_layers=0)
num_test_updates = 5
for u in range(num_test_updates):
mlp.zero_grads()
mlp.forward(x)
mlp.backward(y)
mlp.step()
mlp.eval()
for i, (pred, gt) in enumerate(zip(mlp.W, solW)):
# for i, j in zip(pred, gt):
# for m, n in zip(i, j):
# if (m != n):
# print(m - n,"Momentum Difference")
closeness_test(pred, gt, "mlp.W[%d]" % i)
for i, (pred, gt) in enumerate(zip(mlp.b, solb)):
closeness_test(pred, gt, "mlp.b[%d]" % i)
def test_batch_norm_train():
data = saved_data[19]
assert len(data) == 10
x = data[0]
y = data[1]
soldW = data[2:5]
soldb = data[5:8]
soldbeta = data[8]
soldgamma = data[9]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [hw1.Sigmoid(), hw1.Sigmoid(), hw1.Identity()],
weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=1)
mlp.forward(x)
mlp.backward(y)
for i, (pred, gt) in enumerate(zip(mlp.dW, soldW)):
closeness_test(pred, gt, "mlp.dW[%d]" % i)
for i, (pred, gt) in enumerate(zip(mlp.db, soldb)):
closeness_test(pred, gt, "mlp.db[%d]" % i)
closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta")
closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma")
def test_batch_norm_inference():
num_examples = 1000
data = saved_data[20]
assert len(data) == 15
x = data[0]
y = data[1]
soldbeta = data[2]
soldgamma = data[3]
xs = data[4]
solground = data[5:]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [hw1.Sigmoid(), hw1.Sigmoid(), hw1.Identity()],
weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=1)
batch_size = 100
mlp.train()
for b in range(0, 1):
mlp.zero_grads()
mlp.forward(x[b:b + batch_size])
mlp.backward(y[b:b + batch_size])
mlp.step()
assert mlp.bn_layers[0].dbeta.all() == soldbeta.all(), "mlp.bn_layers[0].dbeta ERROR!"
closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta")
closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma")
for b in range(0, num_examples, batch_size):
mlp.eval()
student = mlp.forward(xs[b:b + batch_size])
ground = solground[b // batch_size]
closeness_test(student, ground, "mlp.forward(x)")
def visualize(outpath):
# Configure the training visualization process below
# Change these hyperparameters around to experiment with your implementation
epochs = 200
batch_size = 100
thisdir = os.path.dirname(__file__)
savepath = outpath
train_data_path = os.path.join(thisdir, "../data/train_data.npy")
train_labels_path = os.path.join(thisdir, "../data/train_labels.npy")
val_data_path = os.path.join(thisdir, "../data/val_data.npy")
val_labels_path = os.path.join(thisdir, "../data/val_labels.npy")
test_data_path = os.path.join(thisdir, "../data/test_data.npy")
test_labels_path = os.path.join(thisdir, "../data/test_labels.npy")
dset = (process_dset_partition(
(np.load(train_data_path), np.load(train_labels_path))),
process_dset_partition(
(np.load(val_data_path), np.load(val_labels_path))),
process_dset_partition(
(np.load(test_data_path), np.load(test_labels_path))))
mlp = nn.MLP(
784, 10, [32, 32, 32],
[nn.Sigmoid(), nn.Sigmoid(),
nn.Sigmoid(), nn.Identity()], nn.random_normal_weight_init,
nn.zeros_bias_init, nn.SoftmaxCrossEntropy(), 1e-3)
visualize_training_statistics(mlp, dset, epochs, batch_size, savepath)
print("Saved output to {}".format(savepath))
def test_mystery_hidden_backward2():
data = saved_data[17]
assert len(data) == 14
x = data[0]
y = data[1]
soldW = data[2:8]
soldb = data[8:]
reset_prng()
mlp = hw1.MLP(784,
10, [32, 32, 32, 32, 32], [
hw1.Sigmoid(),
hw1.Sigmoid(),
hw1.Sigmoid(),
hw1.Sigmoid(),
hw1.Sigmoid(),
hw1.Identity()
],
weight_init,
bias_init,
hw1.SoftmaxCrossEntropy(),
0.008,
momentum=0.0,
num_bn_layers=0)
mlp.forward(x)
mlp.backward(y)
for i, (pred, gt) in enumerate(zip(mlp.dW, soldW)):
closeness_test(pred, gt, "mlp.dW[%d]" % i)
for i, (pred, gt) in enumerate(zip(mlp.db, soldb)):
closeness_test(pred, gt, "mlp.db[%d]" % i)
def test_softmax_cross_entropy_derivative():
data = saved_data[1]
x = data[0]
y = data[1]
sol = data[2]
ce = hw1.SoftmaxCrossEntropy()
ce(x, y)
closeness_test(ce.derivative(), sol, "ce.derivative()")
def test_softmax_cross_entropy_forward():
data = saved_data[0]
x = data[0]
y = data[1]
sol = data[2]
ce = hw1.SoftmaxCrossEntropy()
closeness_test(ce(x, y), sol, "ce(x, y)")
def test_linear_classifier_forward():
data = saved_data[2]
x = data[0]
gt = data[1]
reset_prng()
mlp = hw1.MLP(784, 10, [], [hw1.Identity()], weight_init, bias_init,
hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0,
num_bn_layers=0)
pred = mlp.forward(x)
closeness_test(pred, gt, "mlp.forward(x)")
def test_mystery_hidden_forward3():
data = saved_data[15]
x = data[0]
gt = data[1]
reset_prng()
mlp = hw1.MLP(784, 10, [32], [hw1.Sigmoid(), hw1.Identity()],
weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=0)
pred = mlp.forward(x)
closeness_test(pred, gt, "mlp.forward(x)")
def test_linear_classifier_backward():
data = saved_data[3]
x = data[0]
y = data[1]
soldW = data[2]
soldb = data[3]
reset_prng()
mlp = hw1.MLP(784, 10, [], [hw1.Identity()], weight_init, bias_init,
hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0,
num_bn_layers=0)
mlp.forward(x)
mlp.backward(y)
closeness_test(mlp.dW[0], soldW, "mlp.dW")
closeness_test(mlp.db[0], soldb, "mlp.db")
def test_single_hidden_forward():
data = saved_data[11]
x = data[0]
gt = data[1]
reset_prng()
mlp = hw1.MLP(784, 10, [32], [hw1.Sigmoid(), hw1.Identity()],
weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=0)
pred = mlp.forward(x)
# for i,j in zip(pred,gt):
# for m,n in zip(i,j):
# if (m!=n):
# print(m-n)
closeness_test(pred, gt, "mlp.forward(x)")
def test_linear_classifier_step():
data = saved_data[4]
x = data[0]
y = data[1]
solW = data[2]
solb = data[3]
reset_prng()
mlp = hw1.MLP(784, 10, [], [hw1.Identity()], weight_init, bias_init,
hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0,
num_bn_layers=0)
num_test_updates = 5
for u in range(num_test_updates):
mlp.zero_grads()
mlp.forward(x)
mlp.backward(y)
mlp.step()
# for i in range(len(mlp.W[0])):
# print (mlp.W[0][i][0] , solW[i][0])
closeness_test(mlp.W[0], solW, "mlp.W[0]")
closeness_test(mlp.b[0], solb, "mlp.b[0]")
