def make__two_layer_model():
r"""
Designed to be used only with TestFullNetwork class.. variable names etc. are
used in debugging in the testing code
"""
w_node = node.VarNode('w', True)
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b', True)
w2_node = node.VarNode('w2', True)
b2_node = node.VarNode('b2', True)
start_nodes = [w_node, x_node, b_node, ya_node, w2_node, b2_node]
w = np.array([[1, 3], [0, 1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[.5, .7]]).T
w2 = np.array([[.1, .2], [.3, .07]])
b2 = np.array([[.02, .3]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b, 'w2': w2, 'b2': b2}
wx_node = node.MatrixMultiplication(w_node, x_node, "wx")
sum_node = node.MatrixAddition(wx_node, b_node, "wx+b")
sigmoid_node = SigmoidNode(sum_node, "sig")
wx2_node = node.MatrixMultiplication(w2_node, sigmoid_node, "wx2")
sum2_node = node.MatrixAddition(wx2_node, b2_node, "wx2+b2")
l2_node = L2DistanceSquaredNorm(sum2_node, ya_node, "l2")
return var_map, start_nodes, l2_node
def test_basic(self):
w_node = node.VarNode('w')
x_node = node.VarNode('x')
w = np.array([[1, 3, 0], [0, 1, -1]])
x = np.array([[1, -1], [0, 2], [9, 1]])
mult_node = node.MatrixMultiplication(w_node, x_node)
var_map = {'x': x, 'w': w}
x_node.forward(var_map)
self.assertIsNone(mult_node.value())
w_node.forward(var_map)
value = mult_node.value()
expected = w @ x
np.testing.assert_array_almost_equal(expected, value)
print(value)
self.assertIsNotNone(x_node.value())
mult_node.reset_network_fwd()
# Just checking
# Not none because fwd should start from start vars
self.assertIsNotNone(x_node.value())
b_node = node.VarNode('b')
b = np.array([-1, -1])
var_map['b'] = b
sum_node = node.MatrixAddition(mult_node, b_node)
var_nodes = [x_node, w_node, b_node]
for var_node in var_nodes:
var_node.forward(var_map)
expected = expected + b
np.testing.assert_array_almost_equal(expected, sum_node.value())
print(sum_node.value())
def test_matrix_prd(self):
w_node = node.VarNode('w')
x_node = node.VarNode('x')
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
w_grad_expected = np.array([x[:, 0], x[:, 0]])
local_grad = np.array([[1, 1]]).T
x_grad_expected = np.multiply(w, local_grad).sum(axis=0).T
x_grad_expected = np.reshape(x_grad_expected,
(len(x_grad_expected), 1))
mult_node = node.MatrixMultiplication(w_node, x_node, name="wx")
var_map = {'x': x, 'w': w}
x_node.forward(var_map)
self.assertIsNone(mult_node.value())
w_node.forward(var_map)
value = mult_node.value()
expected = w @ x
np.testing.assert_array_almost_equal(expected, value)
mult_node.backward(local_grad, self, var_map)
w_grad = w_node.total_incoming_gradient()
print("---- printing w_grad ---")
print(w_grad)
np.testing.assert_array_almost_equal(w_grad, w_grad_expected)
print("---- end printing ----")
x_grad = x_node.total_incoming_gradient()
print("---- printing x_grad ---")
print(x_grad)
np.testing.assert_array_almost_equal(x_grad_expected, x_grad)
print("---- end printing ----")
def test_full_sgmoid_node(self):
w_node = node.VarNode('w', True)
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b', True)
start_nodes = [w_node, x_node, b_node, ya_node]
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[.5, .7]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b}
wx_node = node.MatrixMultiplication(w_node, x_node)
sum_node = node.MatrixAddition(wx_node, b_node)
sigmoid_node = SigmoidNode(sum_node)
l2_node = L2DistanceSquaredNorm(sigmoid_node, ya_node)
optim_func = self.rate_adjustable_optimizer_func(0.01)
optimizer = core.np.Optimization.OptimizerIterator(
start_nodes, l2_node, optim_func)
log_at_info()
losses = []
for i in range(100):
loss = optimizer.step(var_map, 1.0)
losses.append(loss)
if i % 10 == 0:
print("[{}] Loss:{}".format(i, loss))
print("Final loss:{}".format(loss))
print("w:{}".format(var_map['w']))
print("b:{}".format(var_map['b']))
def test_linear_transformation(self):
np.random.seed(100)
w_node = node.VarNode('w')
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
w = np.array([[1, 2, 1], [2, 0, -1]])
x = np.array(
[[0.54340494, 0.27836939, 0.42451759, 0.84477613, 0.00471886],
[0.12156912, 0.67074908, 0.82585276, 0.13670659, 0.57509333],
[0.89132195, 0.20920212, 0.18532822, 0.10837689, 0.21969749]])
y_act = np.array(
[[0.97862378, 0.81168315, 0.17194101, 0.81622475, 0.27407375],
[0.43170418, 0.94002982, 0.81764938, 0.33611195, 0.17541045]])
info("Printing x...")
info(x)
info("printing y_pred")
info(y_act)
var_map = {'w': w, 'x': x, 'y_a': y_act}
wx_node = node.MatrixMultiplication(w_node, x_node)
l2_node = L2DistanceSquaredNorm(wx_node, ya_node)
log_at_info()
w_node.forward(var_map)
x_node.forward(var_map)
ya_node.forward(var_map)
l2_node.backward(1.0, self, var_map)
info(wx_node.value())
info("grad...")
info(wx_node.total_incoming_gradient())
def test_single_step(self):
w_node = node.VarNode('w')
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b')
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[1, 2]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b}
wx_node = node.MatrixMultiplication(w_node, x_node)
sum_node = node.MatrixAddition(wx_node, b_node)
l2_node = L2DistanceSquaredNorm(sum_node, ya_node)
w_node.forward(var_map)
x_node.forward(var_map)
b_node.forward(var_map)
ya_node.forward(var_map)
l2norm = l2_node.value()
info("L2Norm: {}".format(l2norm))
y_p = w @ x + b
y_del = y_p - y_act
expected = np.sum(np.square(y_del)) / y_del.size
debug("l2norm:{}".format(l2_node))
self.assertEqual(expected, l2norm)
l2_node.backward(1.0, self, var_map)
w_grad = w_node.total_incoming_gradient()
b_grad = b_node.total_incoming_gradient()
debug("----- w grad ------")
debug(w_grad)
debug("-------------------")
debug("----- b grad ------")
debug(b_grad)
debug("-------------------")
l2_node.reset_network_back()
self.assertIsNone(w_node.total_incoming_gradient())
l2_node.backward(1.0, self, var_map)
w_grad = w_node.total_incoming_gradient()
b_grad = b_node.total_incoming_gradient()
debug("----- w grad ------")
debug(w_grad)
debug("-------------------")
debug("----- b grad ------")
debug(b_grad)
debug("-------------------")
def test_network_optimizer(self):
w_node = node.VarNode('w', True)
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b', True)
start_nodes = [w_node, x_node, b_node, ya_node]
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[1, 2]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b}
wx_node = node.MatrixMultiplication(w_node, x_node)
sum_node = node.MatrixAddition(wx_node, b_node)
l2_node = L2DistanceSquaredNorm(sum_node, ya_node)
optimizer = optim.OptimizerIterator(start_nodes, l2_node)
log_at_info()
for _ in range(500):
loss = optimizer.step(var_map, 1.0)
info("Final loss:{}".format(loss))
self.assertTrue(math.fabs(loss) < 1e-25)