def test_shape_dim2(self):
X = numpy.random.randn(10, 3)
w = numpy.array([[10, 20, 3], [-10, -20, 0.5]])
for act in ['sigmoid', 'sigmoid4', 'expit', 'identity',
'relu', 'leakyrelu']:
with self.subTest(act=act):
neu = NeuralTreeNode(w, bias=[-4, 0.5], activation=act)
pred = neu.predict(X)
self.assertEqual(pred.shape, (X.shape[0], 2))
def test_neural_tree_node(self):
self.assertRaise(lambda: NeuralTreeNode([0, 1], 0.5, 'identity2'))
neu = NeuralTreeNode([0, 1], 0.5, 'identity')
res = neu.predict(numpy.array([4, 5]))
self.assertEqual(res, 5.5)
st = repr(neu)
self.assertEqual("NeuralTreeNode(weights=array([0., 1.]), "
"bias=0.5, activation='identity')", st)
st = io.BytesIO()
pickle.dump(neu, st)
st = io.BytesIO(st.getvalue())
neu2 = pickle.load(st)
self.assertTrue(neu == neu2)
def test_neural_tree_network_append(self):
net = NeuralTreeNet(3, empty=False)
self.assertRaise(lambda: net.append(
NeuralTreeNode(2, activation='identity'), inputs=[3]))
net.append(NeuralTreeNode(1, activation='identity'), inputs=[3])
self.assertEqual(net.size_, 5)
last_node = net.nodes[-1]
X = numpy.random.randn(2, 3)
got = net.predict(X)
exp = X.sum(axis=1) * last_node.input_weights[0] + last_node.bias
self.assertEqual(exp.reshape((-1, 1)), got[:, -1:])
rep = repr(net)
self.assertEqual(rep, 'NeuralTreeNet(3)')
def test_neural_tree_network_copy(self):
net = NeuralTreeNet(3, empty=False)
net.append(NeuralTreeNode(1, activation='identity'),
inputs=[3])
net2 = net.copy()
X = numpy.random.randn(2, 3)
self.assertEqualArray(net.predict(X), net2.predict(X))
def test_neural_tree_network_training_weights(self):
net = NeuralTreeNet(3, empty=False)
net.append(NeuralTreeNode(1, activation='identity'), inputs=[3])
w = net.training_weights
self.assertEqual(w.shape, (6, ))
self.assertEqual(w[0], 0)
self.assertEqualArray(w[1:4], [1, 1, 1])
delta = numpy.arange(6) - 0.5
net.update_training_weights(delta)
w2 = net.training_weights
self.assertEqualArray(w2, w + delta)
def test_neural_net_gradient_regression(self):
X = numpy.abs(numpy.random.randn(10, 2))
w1 = numpy.array([-0.5, 0.8, -0.6])
noise = numpy.random.randn(X.shape[0]) / 10
noise[0] = 0
noise[1] = 0.07
X[1, 0] = 0.7
X[1, 1] = -0.5
y = w1[0] + X[:, 0] * w1[1] + X[:, 1] * w1[2] + noise
for act in ['identity', 'relu', 'leakyrelu',
'sigmoid', 'sigmoid4', 'expit']:
with self.subTest(act=act):
neu = NeuralTreeNode(w1[1:], bias=w1[0], activation=act)
loss1 = neu.loss(X, y)
grad1 = neu.gradient(X[0], y[0])
net = NeuralTreeNet(X.shape[1], empty=True)
net.append(neu, numpy.arange(0, 2))
loss2 = net.loss(X, y)
grad2 = net.gradient(X[0], y[0])
self.assertEqualArray(loss1, loss2)
self.assertEqualArray(grad1, grad2)
def test_shape_dim2(self):
X = numpy.random.randn(10, 3)
w = numpy.array([[10, 20, 3], [-10, -20, 0.5]])
first = None
for act in [
'sigmoid', 'sigmoid4', 'expit', 'identity', 'relu', 'leakyrelu'
]:
with self.subTest(act=act):
neu = NeuralTreeNode(w, bias=[-4, 0.5], activation=act)
pred = neu.predict(X)
self.assertEqual(pred.shape, (X.shape[0], 2))
text = str(neu)
self.assertIn('NeuralTreeNode(', text)
if first is None:
first = neu
else:
self.assertFalse(neu == first)
self.assertEqual(neu.ndim, 3)
loss = neu.loss(X[0], 0.)
self.assertEqual(loss.shape, (2, ))
loss = neu.loss(
X, numpy.zeros((X.shape[0], 1), dtype=numpy.float64))
self.assertEqual(loss.shape, (10, 2))
def test_neural_net_gradient_regression_2(self):
X = numpy.abs(numpy.random.randn(10, 2))
w1 = numpy.array([-0.5, 0.8, -0.6])
noise = numpy.random.randn(X.shape[0]) / 10
noise[0] = 0
noise[1] = 0.07
X[1, 0] = 0.7
X[1, 1] = -0.5
y = w1[0] + X[:, 0] * w1[1] + X[:, 1] * w1[2] + noise
for act in [
'relu', 'sigmoid', 'identity', 'leakyrelu', 'sigmoid4', 'expit'
]:
with self.subTest(act=act):
neu = NeuralTreeNode(w1[1:], bias=w1[0], activation=act)
loss1 = neu.loss(X, y)
pred1 = neu.predict(X)
if act == 'relu':
self.assertEqualArray(pred1[1:2], numpy.array([0.36]))
pred11 = neu.predict(X)
self.assertEqualArray(pred11[1:2], numpy.array([0.36]))
net = NeuralTreeNet(X.shape[1], empty=True)
net.append(neu, numpy.arange(0, 2))
ide = NeuralTreeNode(numpy.array([1], dtype=X.dtype),
bias=numpy.array([0], dtype=X.dtype),
activation='identity')
net.append(ide, numpy.arange(2, 3))
pred2 = net.predict(X)
loss2 = net.loss(X, y)
self.assertEqualArray(pred1, pred2[:, -1])
self.assertEqualArray(pred2[:, -2], pred2[:, -1])
self.assertEqualArray(pred2[:, 2], pred2[:, 3])
self.assertEqualArray(loss1, loss2)
for p in range(0, 5):
grad1 = neu.gradient(X[p], y[p])
grad2 = net.gradient(X[p], y[p])
self.assertEqualArray(grad1, grad2[:3])
def test_optim_clas(self):
X = numpy.abs(numpy.random.randn(10, 2))
w1 = numpy.array([[0.1, 0.8, -0.6], [-0.1, 0.4, -0.3]])
w0 = numpy.random.randn(*w1.shape)
noise = numpy.random.randn(*X.shape) / 10
noise[0] = 0
noise[1] = 0.07
y0 = (X[:, :1] @ w1[:, 1:2].T +
X[:, 1:] @ w1[:, 2:3].T + w1[:, 0].T + noise)
y = numpy.exp(y0)
y /= numpy.sum(y, axis=1, keepdims=1)
y[:-1, 0] = (y[:-1, 0] >= 0.5).astype(numpy.float64)
y[:-1, 1] = (y[:-1, 1] >= 0.5).astype(numpy.float64)
y /= numpy.sum(y, axis=1, keepdims=1)
for act in ['softmax', 'softmax4']:
with self.subTest(act=act):
neu2 = NeuralTreeNode(2, activation=act)
neu = NeuralTreeNode(w1[:, 1:], bias=w1[:, 0], activation=act)
self.assertEqual(neu2.training_weights.shape,
neu.training_weights.shape)
self.assertEqual(neu2.input_weights.shape,
neu.input_weights.shape)
loss = neu.loss(X, y).sum() / X.shape[0]
self.assertNotEmpty(loss)
self.assertFalse(numpy.isinf(loss))
self.assertFalse(numpy.isnan(loss))
grad = neu.gradient(X[0], y[0])
self.assertEqual(grad.ravel().shape, w1.ravel().shape)
neu.fit(X, y, verbose=False)
c1 = neu.training_weights
neu = NeuralTreeNode(w0[:, 1:], bias=w0[:, 0], activation=act)
neu.fit(X, y, verbose=False, lr_schedule='constant')
c2 = neu.training_weights
self.assertEqual(c1.shape, c2.shape)
def test_optim_regression(self):
X = numpy.abs(numpy.random.randn(10, 2))
w0 = numpy.random.randn(3)
w1 = numpy.array([-0.5, 0.8, -0.6])
noise = numpy.random.randn(X.shape[0]) / 10
noise[0] = 0
noise[1] = 0.07
X[1, 0] = 0.7
X[1, 1] = -0.5
y = w1[0] + X[:, 0] * w1[1] + X[:, 1] * w1[2] + noise
for act in ['identity', 'relu', 'leakyrelu',
'sigmoid', 'sigmoid4', 'expit']:
with self.subTest(act=act):
neu = NeuralTreeNode(w1[1:], bias=w1[0], activation=act)
loss = neu.loss(X, y).sum() / X.shape[0]
if act == 'identity':
self.assertGreater(loss, 0)
self.assertLess(loss, 0.1)
grad = neu.gradient(X[0], y[0])
if act == 'identity':
self.assertEqualArray(grad, numpy.zeros(grad.shape))
grad = neu.gradient(X[1], y[1])
ming, maxg = grad[:2].min(), grad[:2].max()
if ming == maxg:
raise AssertionError(
"Gradient is wrong\nloss={}\ngrad={}".format(
loss, grad))
self.assertEqual(grad.shape, w0.shape)
neu.fit(X, y, verbose=False)
c1 = neu.training_weights
neu = NeuralTreeNode(w0[1:], bias=w0[0], activation=act)
neu.fit(X, y, verbose=False, lr_schedule='constant')
c2 = neu.training_weights
diff = numpy.abs(c2 - c1)
if act == 'identity':
self.assertLess(diff.max(), 0.16)
def test_gradients(self):
X = numpy.array([0.1, 0.2, -0.3])
w = numpy.array([10, 20, 3])
g = numpy.array([-0.7], dtype=numpy.float64)
for act in ['sigmoid', 'sigmoid4', 'expit', 'identity',
'relu', 'leakyrelu']:
with self.subTest(act=act):
neu = NeuralTreeNode(w, bias=-4, activation=act)
pred = neu.predict(X)
self.assertEqual(pred.shape, tuple())
grad = neu.gradient_backward(g, X)
self.assertEqual(grad.shape, (4, ))
grad = neu.gradient_backward(g, X, inputs=True)
self.assertEqual(grad.shape, (3, ))
ww = neu.training_weights
neu.update_training_weights(-ww)
w0 = neu.training_weights
self.assertEqualArray(w0, numpy.zeros(w0.shape))
X = numpy.array([0.1, 0.2, -0.3])
w = numpy.array([[10, 20, 3], [-10, -20, 3]])
b = numpy.array([-3, 4], dtype=numpy.float64)
g = numpy.array([-0.7, 0.2], dtype=numpy.float64)
for act in ['softmax', 'softmax4']:
with self.subTest(act=act):
neu = NeuralTreeNode(w, bias=b, activation=act)
pred = neu.predict(X)
self.assertAlmostEqual(numpy.sum(pred), 1.)
self.assertEqual(pred.shape, (2,))
grad = neu.gradient_backward(g, X)
self.assertEqual(grad.shape, (2, 4))
grad = neu.gradient_backward(g, X, inputs=True)
self.assertEqual(grad.shape, (3, ))
ww = neu.training_weights
neu.update_training_weights(-ww)
w0 = neu.training_weights
self.assertEqualArray(w0, numpy.zeros(w0.shape))