def test_momentum(): data = saved_data[21] assert len(data) == 8 x = data[0] y = data[1] solW = data[2:5] solb = data[5:] reset_prng() mlp = hw1.MLP(784, 10, [64, 32], [hw1.Sigmoid(), hw1.Sigmoid(), hw1.Identity()], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.856, num_bn_layers=0) num_test_updates = 5 for u in range(num_test_updates): mlp.zero_grads() mlp.forward(x) mlp.backward(y) mlp.step() mlp.eval() for i, (pred, gt) in enumerate(zip(mlp.W, solW)): # for i, j in zip(pred, gt): # for m, n in zip(i, j): # if (m != n): # print(m - n,"Momentum Difference") closeness_test(pred, gt, "mlp.W[%d]" % i) for i, (pred, gt) in enumerate(zip(mlp.b, solb)): closeness_test(pred, gt, "mlp.b[%d]" % i)
def test_batch_norm_train(): data = saved_data[19] assert len(data) == 10 x = data[0] y = data[1] soldW = data[2:5] soldb = data[5:8] soldbeta = data[8] soldgamma = data[9] reset_prng() mlp = hw1.MLP(784, 10, [64, 32], [hw1.Sigmoid(), hw1.Sigmoid(), hw1.Identity()], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=1) mlp.forward(x) mlp.backward(y) for i, (pred, gt) in enumerate(zip(mlp.dW, soldW)): closeness_test(pred, gt, "mlp.dW[%d]" % i) for i, (pred, gt) in enumerate(zip(mlp.db, soldb)): closeness_test(pred, gt, "mlp.db[%d]" % i) closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta") closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma")
def test_batch_norm_inference(): num_examples = 1000 data = saved_data[20] assert len(data) == 15 x = data[0] y = data[1] soldbeta = data[2] soldgamma = data[3] xs = data[4] solground = data[5:] reset_prng() mlp = hw1.MLP(784, 10, [64, 32], [hw1.Sigmoid(), hw1.Sigmoid(), hw1.Identity()], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=1) batch_size = 100 mlp.train() for b in range(0, 1): mlp.zero_grads() mlp.forward(x[b:b + batch_size]) mlp.backward(y[b:b + batch_size]) mlp.step() assert mlp.bn_layers[0].dbeta.all() == soldbeta.all(), "mlp.bn_layers[0].dbeta ERROR!" closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta") closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma") for b in range(0, num_examples, batch_size): mlp.eval() student = mlp.forward(xs[b:b + batch_size]) ground = solground[b // batch_size] closeness_test(student, ground, "mlp.forward(x)")
def visualize(outpath): # Configure the training visualization process below # Change these hyperparameters around to experiment with your implementation epochs = 200 batch_size = 100 thisdir = os.path.dirname(__file__) savepath = outpath train_data_path = os.path.join(thisdir, "../data/train_data.npy") train_labels_path = os.path.join(thisdir, "../data/train_labels.npy") val_data_path = os.path.join(thisdir, "../data/val_data.npy") val_labels_path = os.path.join(thisdir, "../data/val_labels.npy") test_data_path = os.path.join(thisdir, "../data/test_data.npy") test_labels_path = os.path.join(thisdir, "../data/test_labels.npy") dset = (process_dset_partition( (np.load(train_data_path), np.load(train_labels_path))), process_dset_partition( (np.load(val_data_path), np.load(val_labels_path))), process_dset_partition( (np.load(test_data_path), np.load(test_labels_path)))) mlp = nn.MLP( 784, 10, [32, 32, 32], [nn.Sigmoid(), nn.Sigmoid(), nn.Sigmoid(), nn.Identity()], nn.random_normal_weight_init, nn.zeros_bias_init, nn.SoftmaxCrossEntropy(), 1e-3) visualize_training_statistics(mlp, dset, epochs, batch_size, savepath) print("Saved output to {}".format(savepath))
def test_mystery_hidden_backward2(): data = saved_data[17] assert len(data) == 14 x = data[0] y = data[1] soldW = data[2:8] soldb = data[8:] reset_prng() mlp = hw1.MLP(784, 10, [32, 32, 32, 32, 32], [ hw1.Sigmoid(), hw1.Sigmoid(), hw1.Sigmoid(), hw1.Sigmoid(), hw1.Sigmoid(), hw1.Identity() ], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) mlp.forward(x) mlp.backward(y) for i, (pred, gt) in enumerate(zip(mlp.dW, soldW)): closeness_test(pred, gt, "mlp.dW[%d]" % i) for i, (pred, gt) in enumerate(zip(mlp.db, soldb)): closeness_test(pred, gt, "mlp.db[%d]" % i)
def test_softmax_cross_entropy_derivative(): data = saved_data[1] x = data[0] y = data[1] sol = data[2] ce = hw1.SoftmaxCrossEntropy() ce(x, y) closeness_test(ce.derivative(), sol, "ce.derivative()")
def test_softmax_cross_entropy_forward(): data = saved_data[0] x = data[0] y = data[1] sol = data[2] ce = hw1.SoftmaxCrossEntropy() closeness_test(ce(x, y), sol, "ce(x, y)")
def test_linear_classifier_forward(): data = saved_data[2] x = data[0] gt = data[1] reset_prng() mlp = hw1.MLP(784, 10, [], [hw1.Identity()], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) pred = mlp.forward(x) closeness_test(pred, gt, "mlp.forward(x)")
def test_mystery_hidden_forward3(): data = saved_data[15] x = data[0] gt = data[1] reset_prng() mlp = hw1.MLP(784, 10, [32], [hw1.Sigmoid(), hw1.Identity()], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) pred = mlp.forward(x) closeness_test(pred, gt, "mlp.forward(x)")
def test_linear_classifier_backward(): data = saved_data[3] x = data[0] y = data[1] soldW = data[2] soldb = data[3] reset_prng() mlp = hw1.MLP(784, 10, [], [hw1.Identity()], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) mlp.forward(x) mlp.backward(y) closeness_test(mlp.dW[0], soldW, "mlp.dW") closeness_test(mlp.db[0], soldb, "mlp.db")
def test_single_hidden_forward(): data = saved_data[11] x = data[0] gt = data[1] reset_prng() mlp = hw1.MLP(784, 10, [32], [hw1.Sigmoid(), hw1.Identity()], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) pred = mlp.forward(x) # for i,j in zip(pred,gt): # for m,n in zip(i,j): # if (m!=n): # print(m-n) closeness_test(pred, gt, "mlp.forward(x)")
def test_linear_classifier_step(): data = saved_data[4] x = data[0] y = data[1] solW = data[2] solb = data[3] reset_prng() mlp = hw1.MLP(784, 10, [], [hw1.Identity()], weight_init, bias_init, hw1.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) num_test_updates = 5 for u in range(num_test_updates): mlp.zero_grads() mlp.forward(x) mlp.backward(y) mlp.step() # for i in range(len(mlp.W[0])): # print (mlp.W[0][i][0] , solW[i][0]) closeness_test(mlp.W[0], solW, "mlp.W[0]") closeness_test(mlp.b[0], solb, "mlp.b[0]")