def test_fwd_linear(): model = NeuralNet((2,2), rho=0.0, alpha=0.5) assert model.nr_class == 2 assert model.widths == (2, 2) syn = [1.,0.,0.,1.] bias = [0.,0.] gamma = [1.,1.] if model.use_batch_norm: model.weights = syn+bias+gamma else: model.weights = syn+bias ff = [0,0] tf = [1,0] ft = [0,1] tt = [1,1] eg = model.predict_dense(ff) assert_allclose(eg.scores, [0.5, 0.5]) eg = model.predict_dense(ft) assert_allclose(eg.scores, [ 0.26894142, 0.73105858]) assert_allclose([sum(eg.scores)], [1.0]) eg = model.predict_dense(tf) assert_allclose(eg.scores, [0.73105858, 0.26894142]) assert_allclose(sum(eg.scores), [1.0]) eg = model.predict_dense(tt) assert_allclose(eg.scores, [0.5, 0.5])
def test_xor_manual(): model = NeuralNet((2,2,2), rho=0.0) assert model.nr_class == 2 assert model.widths == (2, 2, 2) # Make a network that detects X-or # It should output 0 if inputs are 0,0 or 1,1 and 1 if inputs are 0,1 or 1,0 # A linear model can't do this! # # What we do is create two intermediate predictors, for 0,1 and 1,0 # These predictors rely on a bias towards 0. The non-linearity is essential # Then our output layer can detect either of these signals firing # # 0,0 --> neither fire # 0,1 --> A0 fires # 1,0 --> A1 fires # 1,1 --> neither fire # if not model.use_batch_norm: model.weights = np.asarray([ [4.0, -10.0], # A.0*in.0, A.0*in.1 [-10.0, 5.0], # A.1*in.0, A.1*in.1 [0.0, 0.0], # A.0 bias, A.1 bias [-10.0, -10.0], # out.0*A.0, out.0*A.1 [10.0, 10.0], # out.1*A.0, out.1*A.1 [10.0, -10.0], # out.0 bias, out.1 bias ]).flatten() else: model.weights = np.asarray([ [4.0, -10.0], # A.0*in.0, A.0*in.1 [-10.0, 5.0], # A.1*in.0, A.1*in.1 [0.0, 0.0], # A.0 bias, A.1 bias [1.0, 1.0], # A.0 gamma, A.1 gamma [-10.0, -10.0], # out.0*A.0, out.0*A.1 [10.0, 10.0], # out.1*A.0, out.1*A.1 [10.0, -10.0], # out.0 bias, out.1 bias [1.0, 1.0], # out.0 gamma, out.1 gamma ]).flatten() ff = [0,0] tf = [1,0] ft = [0,1] tt = [1,1] eg = model.predict_dense(ff) assert eg.scores[0] > 0.99 eg = model.predict_dense(tt) assert eg.scores[0] > 0.99 eg = model.predict_dense(tf) assert eg.scores[1] > 0.99 eg = model.predict_dense(ft) assert eg.scores[1] > 0.99
def test_xor_gradient(xor_data): '''Test that after each update, we move towards the correct label.''' model = NeuralNet((2, 2, 2), rho=0.0, eta=0.1, update_step='adadelta') assert model.nr_in == 2 assert model.nr_class == 2 assert model.nr_layer == 3 for _ in range(500): for i, (features, label, costs) in enumerate(xor_data): prev = list(model.predict_dense(features).scores) assert_allclose([sum(prev)], [1.0]) tmp = model.train_dense(features, costs) eg = model.predict_dense(features) assert (prev[label] <= eg.scores[label] or \ prev[label] == eg.scores[label] == 1.0)
def test_deep_bias(bias_data): '''Test that a deep model can learn a bias.''' model = NeuralNet((2,2,2,2,2,2,2, 2), rho=0.0, eta=0.1, eps=1e-4, update_step='adadelta') assert model.nr_in == 2 assert model.nr_class == 2 assert model.nr_layer > 2 if not model.use_batch_norm: bias0, bias1 = model.weights[-2:] else: bias0, bias1 = model.weights[-4:-2] assert bias0 == 0 assert bias1 == 0 for _ in range(20): for feats, label, costs in bias_data(): eg = model.train_dense(feats, costs) if not model.use_batch_norm: bias0, bias1 = model.weights[-2:] else: bias0, bias1 = model.weights[-4:-2] assert bias1 > bias0 acc = 0.0 total = 0 for i in range(20): for features, label, costs in bias_data(): eg = model.predict_dense(features) assert costs[label] == 0 acc += eg.scores[label] > 0.5 total += 1 assert (acc/total) > 0.5 assert (acc/total) < 1.0
def test_linear_bias(bias_data): '''Test that a linear model can learn a bias.''' model = NeuralNet((2, 2), rho=0.0, eta=0.1, update_step='sgd') assert model.nr_in == 2 assert model.nr_class == 2 assert model.nr_layer == 2 bias0, bias1 = model.weights[-2:] assert bias0 == 0 assert bias1 == 0 for _ in range(100): for feats, label, costs in bias_data(): model.train_dense(feats, costs) bias0, bias1 = model.weights[-2:] assert bias1 > bias0 acc = 0.0 total = 0 for i in range(20): for features, label, costs in bias_data(): eg = model.predict_dense(features) assert costs[label] == 0 acc += eg.scores[label] > 0.5 total += 1 assert (acc / total) > 0.5 assert (acc / total) < 1.0
def test_fwd_bias(): model = NeuralNet((2, 2), rho=0.0) model.weights = [1.0] * model.nr_weight eg = model.predict_dense([0, 0]) assert eg.nr_class == 2 assert_allclose(eg.scores, [0.5, 0.5]) # Set bias for class 0 syn = [1.,1.,1.,1.] bias = [100000.,1.] gamma = [0,0] if model.nr_weight == len(syn) + len(bias): model.weights = syn + bias assert model.weights == list(syn + bias) else: model.weights = syn + bias + gamma assert model.weights == list(syn + bias + gamma) eg = model.predict_dense([0, 0]) assert_allclose(eg.scores, [1.0, 0.0]) # Set bias for class 1 if model.nr_weight == 6: model.weights = syn + [1.,10000.0] else: model.weights = syn + [1.,10000.0] + gamma eg = model.predict_dense([0,0]) assert_allclose(eg.scores, [0.0, 1.0]) # Set bias for both if model.nr_weight == 6: model.weights = syn + [10000.0,10000.0] else: model.weights = syn + [10000.0,10000.0] + gamma eg = model.predict_dense([0,0]) assert_allclose(eg.scores, [0.5, 0.5])
def test_xor_deep(xor_data): '''Compare 0, 1 and 3 layer networks. The 3 layer seems to do better, but it doesn't *have* to. But if the 0 layer works, something's wrong!''' linear = NeuralNet((2,2), rho=0.0000, eta=0.1, update_step='sgd') small = NeuralNet((2,2,2), rho=0.0000, eta=0.1, update_step='sgd') big = NeuralNet((2,2,2,2,2,2), rho=0.0000, eta=0.1, update_step='sgd') for _ in range(1000): for i, (features, label, costs) in enumerate(xor_data): ln = linear.train_dense(features, costs) bg = big.train_dense(features, costs) sm = small.train_dense(features, costs) random.shuffle(xor_data) linear_loss = 0.0 small_loss = 0.0 big_loss = 0.0 for i, (features, label, costs) in enumerate(xor_data): linear_loss += 1 - linear.predict_dense(features).scores[label] small_loss += 1 - small.predict_dense(features).scores[label] big_loss += 1 - big.predict_dense(features).scores[label] assert big_loss < 0.5 assert small_loss < 2.0 assert linear_loss > 1.9
def test_learn_linear(or_data): '''Test that a linear model can learn OR.''' # Need high eta on this sort of toy problem, or learning takes forever! model = NeuralNet((2, 2), rho=0.0, eta=0.1, update_step='sgd') assert model.nr_in == 2 assert model.nr_class == 2 assert model.nr_layer == 2 # It takes about this many iterations, with the settings above. for _ in range(900): for feats, label, costs in or_data: eg = model.train_dense(feats, costs) random.shuffle(or_data) acc = 0.0 for features, label, costs in or_data: eg = model.predict_dense(features) assert costs[label] == 0 acc += eg.scores[label] > 0.5 assert acc == len(or_data)
def test_mlp_learn_linear(or_data): '''Test that with a hidden layer, we can still learn OR''' # Need high eta on this sort of toy problem, or learning takes forever! model = NeuralNet((2, 3, 2), rho=0.0, eta=0.5, update_step='sgd') assert model.nr_in == 2 assert model.nr_class == 2 assert model.nr_layer == 3 # Keep this set low, so that we see that the hidden layer allows the function # to be learned faster than the linear model for _ in range(50): for feats, label, costs in or_data: eg = model.train_dense(feats, costs) random.shuffle(or_data) acc = 0.0 for features, label, costs in or_data: eg = model.predict_dense(features) assert costs[label] == 0 acc += eg.scores[label] > 0.5 assert acc == len(or_data)
def test_mlp_learn_linear(or_data): '''Test that with a hidden layer, we can still learn OR''' # Need high eta on this sort of toy problem, or learning takes forever! model = NeuralNet((2, 3, 2), rho=0.0, eta=0.5, eps=1e-4, update_step='sgd') assert model.nr_in == 2 assert model.nr_class == 2 assert model.nr_layer == 3 # Keep this set low, so that we see that the hidden layer allows the function # to be learned faster than the linear model for _ in range(50): for feats, label, costs in or_data: batch = model.train_dense(feats, costs) random.shuffle(or_data) acc = 0.0 for features, label, costs in or_data: eg = model.predict_dense(features) assert costs[label] == 0 acc += eg.scores[label] > 0.5 assert acc == len(or_data)
def test_learn_linear(or_data): '''Test that a linear model can learn OR.''' # Need high eta on this sort of toy problem, or learning takes forever! model = NeuralNet((2, 2), rho=0.0, eta=0.1, eps=1e-4, update_step='sgd', alpha=0.8) assert model.nr_in == 2 assert model.nr_class == 2 assert model.nr_layer == 2 # It takes about this many iterations, with the settings above. for _ in range(900): for feats, label, costs in or_data: eg = model.train_dense(feats, costs) random.shuffle(or_data) for avg in model.averages: print(avg) acc = 0.0 for features, label, costs in or_data: eg = model.predict_dense(features) assert costs[label] == 0 acc += eg.scores[label] > 0.5 assert acc == len(or_data)