def test_fwd_linear():
model = NeuralNet((2,2), rho=0.0, alpha=0.5)
assert model.nr_class == 2
assert model.widths == (2, 2)
syn = [1.,0.,0.,1.]
bias = [0.,0.]
gamma = [1.,1.]
if model.use_batch_norm:
model.weights = syn+bias+gamma
else:
model.weights = syn+bias
ff = [0,0]
tf = [1,0]
ft = [0,1]
tt = [1,1]
eg = model.predict_dense(ff)
assert_allclose(eg.scores, [0.5, 0.5])
eg = model.predict_dense(ft)
assert_allclose(eg.scores, [ 0.26894142, 0.73105858])
assert_allclose([sum(eg.scores)], [1.0])
eg = model.predict_dense(tf)
assert_allclose(eg.scores, [0.73105858, 0.26894142])
assert_allclose(sum(eg.scores), [1.0])
eg = model.predict_dense(tt)
assert_allclose(eg.scores, [0.5, 0.5])
def test_xor_manual():
model = NeuralNet((2,2,2), rho=0.0)
assert model.nr_class == 2
assert model.widths == (2, 2, 2)
# Make a network that detects X-or
# It should output 0 if inputs are 0,0 or 1,1 and 1 if inputs are 0,1 or 1,0
# A linear model can't do this!
#
# What we do is create two intermediate predictors, for 0,1 and 1,0
# These predictors rely on a bias towards 0. The non-linearity is essential
# Then our output layer can detect either of these signals firing
#
# 0,0 --> neither fire
# 0,1 --> A0 fires
# 1,0 --> A1 fires
# 1,1 --> neither fire
#
if not model.use_batch_norm:
model.weights = np.asarray([
[4.0, -10.0], # A.0*in.0, A.0*in.1
[-10.0, 5.0], # A.1*in.0, A.1*in.1
[0.0, 0.0], # A.0 bias, A.1 bias
[-10.0, -10.0], # out.0*A.0, out.0*A.1
[10.0, 10.0], # out.1*A.0, out.1*A.1
[10.0, -10.0], # out.0 bias, out.1 bias
]).flatten()
else:
model.weights = np.asarray([
[4.0, -10.0], # A.0*in.0, A.0*in.1
[-10.0, 5.0], # A.1*in.0, A.1*in.1
[0.0, 0.0], # A.0 bias, A.1 bias
[1.0, 1.0], # A.0 gamma, A.1 gamma
[-10.0, -10.0], # out.0*A.0, out.0*A.1
[10.0, 10.0], # out.1*A.0, out.1*A.1
[10.0, -10.0], # out.0 bias, out.1 bias
[1.0, 1.0], # out.0 gamma, out.1 gamma
]).flatten()
ff = [0,0]
tf = [1,0]
ft = [0,1]
tt = [1,1]
eg = model.predict_dense(ff)
assert eg.scores[0] > 0.99
eg = model.predict_dense(tt)
assert eg.scores[0] > 0.99
eg = model.predict_dense(tf)
assert eg.scores[1] > 0.99
eg = model.predict_dense(ft)
assert eg.scores[1] > 0.99
def test_xor_gradient(xor_data):
'''Test that after each update, we move towards the correct label.'''
model = NeuralNet((2, 2, 2), rho=0.0, eta=0.1, update_step='adadelta')
assert model.nr_in == 2
assert model.nr_class == 2
assert model.nr_layer == 3
for _ in range(500):
for i, (features, label, costs) in enumerate(xor_data):
prev = list(model.predict_dense(features).scores)
assert_allclose([sum(prev)], [1.0])
tmp = model.train_dense(features, costs)
eg = model.predict_dense(features)
assert (prev[label] <= eg.scores[label] or \
prev[label] == eg.scores[label] == 1.0)
def test_deep_bias(bias_data):
'''Test that a deep model can learn a bias.'''
model = NeuralNet((2,2,2,2,2,2,2, 2), rho=0.0, eta=0.1, eps=1e-4, update_step='adadelta')
assert model.nr_in == 2
assert model.nr_class == 2
assert model.nr_layer > 2
if not model.use_batch_norm:
bias0, bias1 = model.weights[-2:]
else:
bias0, bias1 = model.weights[-4:-2]
assert bias0 == 0
assert bias1 == 0
for _ in range(20):
for feats, label, costs in bias_data():
eg = model.train_dense(feats, costs)
if not model.use_batch_norm:
bias0, bias1 = model.weights[-2:]
else:
bias0, bias1 = model.weights[-4:-2]
assert bias1 > bias0
acc = 0.0
total = 0
for i in range(20):
for features, label, costs in bias_data():
eg = model.predict_dense(features)
assert costs[label] == 0
acc += eg.scores[label] > 0.5
total += 1
assert (acc/total) > 0.5
assert (acc/total) < 1.0
def test_linear_bias(bias_data):
'''Test that a linear model can learn a bias.'''
model = NeuralNet((2, 2), rho=0.0, eta=0.1, update_step='sgd')
assert model.nr_in == 2
assert model.nr_class == 2
assert model.nr_layer == 2
bias0, bias1 = model.weights[-2:]
assert bias0 == 0
assert bias1 == 0
for _ in range(100):
for feats, label, costs in bias_data():
model.train_dense(feats, costs)
bias0, bias1 = model.weights[-2:]
assert bias1 > bias0
acc = 0.0
total = 0
for i in range(20):
for features, label, costs in bias_data():
eg = model.predict_dense(features)
assert costs[label] == 0
acc += eg.scores[label] > 0.5
total += 1
assert (acc / total) > 0.5
assert (acc / total) < 1.0
def test_fwd_bias():
model = NeuralNet((2, 2), rho=0.0)
model.weights = [1.0] * model.nr_weight
eg = model.predict_dense([0, 0])
assert eg.nr_class == 2
assert_allclose(eg.scores, [0.5, 0.5])
# Set bias for class 0
syn = [1.,1.,1.,1.]
bias = [100000.,1.]
gamma = [0,0]
if model.nr_weight == len(syn) + len(bias):
model.weights = syn + bias
assert model.weights == list(syn + bias)
else:
model.weights = syn + bias + gamma
assert model.weights == list(syn + bias + gamma)
eg = model.predict_dense([0, 0])
assert_allclose(eg.scores, [1.0, 0.0])
# Set bias for class 1
if model.nr_weight == 6:
model.weights = syn + [1.,10000.0]
else:
model.weights = syn + [1.,10000.0] + gamma
eg = model.predict_dense([0,0])
assert_allclose(eg.scores, [0.0, 1.0])
# Set bias for both
if model.nr_weight == 6:
model.weights = syn + [10000.0,10000.0]
else:
model.weights = syn + [10000.0,10000.0] + gamma
eg = model.predict_dense([0,0])
assert_allclose(eg.scores, [0.5, 0.5])
def test_xor_deep(xor_data):
'''Compare 0, 1 and 3 layer networks.
The 3 layer seems to do better, but it doesn't *have* to. But if the
0 layer works, something's wrong!'''
linear = NeuralNet((2,2), rho=0.0000, eta=0.1, update_step='sgd')
small = NeuralNet((2,2,2), rho=0.0000, eta=0.1, update_step='sgd')
big = NeuralNet((2,2,2,2,2,2), rho=0.0000, eta=0.1, update_step='sgd')
for _ in range(1000):
for i, (features, label, costs) in enumerate(xor_data):
ln = linear.train_dense(features, costs)
bg = big.train_dense(features, costs)
sm = small.train_dense(features, costs)
random.shuffle(xor_data)
linear_loss = 0.0
small_loss = 0.0
big_loss = 0.0
for i, (features, label, costs) in enumerate(xor_data):
linear_loss += 1 - linear.predict_dense(features).scores[label]
small_loss += 1 - small.predict_dense(features).scores[label]
big_loss += 1 - big.predict_dense(features).scores[label]
assert big_loss < 0.5
assert small_loss < 2.0
assert linear_loss > 1.9
def test_learn_linear(or_data):
'''Test that a linear model can learn OR.'''
# Need high eta on this sort of toy problem, or learning takes forever!
model = NeuralNet((2, 2), rho=0.0, eta=0.1, update_step='sgd')
assert model.nr_in == 2
assert model.nr_class == 2
assert model.nr_layer == 2
# It takes about this many iterations, with the settings above.
for _ in range(900):
for feats, label, costs in or_data:
eg = model.train_dense(feats, costs)
random.shuffle(or_data)
acc = 0.0
for features, label, costs in or_data:
eg = model.predict_dense(features)
assert costs[label] == 0
acc += eg.scores[label] > 0.5
assert acc == len(or_data)
def test_mlp_learn_linear(or_data):
'''Test that with a hidden layer, we can still learn OR'''
# Need high eta on this sort of toy problem, or learning takes forever!
model = NeuralNet((2, 3, 2), rho=0.0, eta=0.5, update_step='sgd')
assert model.nr_in == 2
assert model.nr_class == 2
assert model.nr_layer == 3
# Keep this set low, so that we see that the hidden layer allows the function
# to be learned faster than the linear model
for _ in range(50):
for feats, label, costs in or_data:
eg = model.train_dense(feats, costs)
random.shuffle(or_data)
acc = 0.0
for features, label, costs in or_data:
eg = model.predict_dense(features)
assert costs[label] == 0
acc += eg.scores[label] > 0.5
assert acc == len(or_data)
def test_mlp_learn_linear(or_data):
'''Test that with a hidden layer, we can still learn OR'''
# Need high eta on this sort of toy problem, or learning takes forever!
model = NeuralNet((2, 3, 2), rho=0.0, eta=0.5, eps=1e-4,
update_step='sgd')
assert model.nr_in == 2
assert model.nr_class == 2
assert model.nr_layer == 3
# Keep this set low, so that we see that the hidden layer allows the function
# to be learned faster than the linear model
for _ in range(50):
for feats, label, costs in or_data:
batch = model.train_dense(feats, costs)
random.shuffle(or_data)
acc = 0.0
for features, label, costs in or_data:
eg = model.predict_dense(features)
assert costs[label] == 0
acc += eg.scores[label] > 0.5
assert acc == len(or_data)
def test_learn_linear(or_data):
'''Test that a linear model can learn OR.'''
# Need high eta on this sort of toy problem, or learning takes forever!
model = NeuralNet((2, 2), rho=0.0, eta=0.1, eps=1e-4, update_step='sgd',
alpha=0.8)
assert model.nr_in == 2
assert model.nr_class == 2
assert model.nr_layer == 2
# It takes about this many iterations, with the settings above.
for _ in range(900):
for feats, label, costs in or_data:
eg = model.train_dense(feats, costs)
random.shuffle(or_data)
for avg in model.averages:
print(avg)
acc = 0.0
for features, label, costs in or_data:
eg = model.predict_dense(features)
assert costs[label] == 0
acc += eg.scores[label] > 0.5
assert acc == len(or_data)
