class LSTM(GenericLayer):
def __init__(self, input_size, output_size):
self.input_size = input_size
self.output_size = output_size
vars = ['xh', 'c']
xh = Input(vars, 'xh')
c = Input(vars, 'c')
Wi = MatrixWeight(input_size + output_size, output_size)
Wf = MatrixWeight(input_size + output_size, output_size)
Wc = MatrixWeight(input_size + output_size, output_size)
bf = Weight(output_size)
bi = Weight(output_size)
bc = Weight(output_size)
self.ct_net = ComputationalGraphLayer(
Sigmoid(Wf * xh + bf) * c +
Sigmoid(Wi * xh + bi) * Tanh(Wc * xh + bc))
Wo = MatrixWeight(input_size + output_size, output_size)
bo = Weight(output_size)
self.ht_net = ComputationalGraphLayer(Tanh(c) * (Wo * xh + bo))
self.ct = np.zeros(output_size)
self.ht = np.zeros(output_size)
def forward(self, x, update=False):
xh = np.hstack([x, self.ht])
self.ct = self.ct_net.forward([xh, self.ct])
self.ht = self.ht_net.forward([xh, self.ct])
return self.ht
def backward(self, dJdy, optimizer=None):
dJdx_group = self.ht_net.backward(dJdy, optimizer)
dJdx = self.ct_net.backward(dJdx_group[0], optimizer)
return dJdx
def test_one_input(self):
x = Input(['x'], 'x')
net = ComputationalGraphLayer(x * x * 3.0 + x * 4.3 + 3.1 - 1 - x *
(x**3 + x * 1.2 + 2) + 1)
xv = np.array([1.3])
out = net.forward(xv)
self.assertEqual(out.shape, (1, ))
assert_almost_equal(
out,
np.array(xv * xv * 3 + xv * 4.3 + 3.1 - 1 - xv *
(xv**3 + xv * 1.2 + 2) + 1))
dJdx = net.backward(np.array([1.0]))
self.assertEqual(dJdx.shape, (1, ))
assert_almost_equal(dJdx, -4 * (-0.575 - 0.9 * xv + xv**3))
xv = np.array([1.1, 2.0, 3.0, 4.0])
out = net.forward(xv)
self.assertEqual(out.shape, (4, ))
assert_almost_equal(
out,
np.array(xv * xv * 3 + xv * 4.3 + 3.1 - 1 - xv *
(xv**3 + xv * 1.2 + 2) + 1))
dJdx = net.backward(np.array([1.0, 1.0, 1.0, 1.0]))
self.assertEqual(dJdx.shape, (4, ))
assert_almost_equal(dJdx, -4 * (-0.575 - 0.9 * xv + xv**3))
net = ComputationalGraphLayer(x - x)
xv = np.array([1.3])
out = net.forward(xv)
self.assertEqual(out.shape, (1, ))
assert_almost_equal(out, np.array(xv - xv))
def test_variable_weight_one_input(self):
#function x**2*a+x*b+c
xv = np.array([1.3])
x = Input(['x'], 'x')
av = np.array([2.1])
bv = np.array([3.2])
cv = np.array([5.1])
a = VWeight(1, weights=av)
b = VWeight(1, weights=bv)
c = VWeight(1, weights=cv)
net = ComputationalGraphLayer(a * x**2 + b * x + c)
out = net.forward(xv)
self.assertEqual(out.shape, (1, ))
assert_almost_equal(out, xv**2 * av + xv * bv + cv)
dJdy = net.backward(np.array([1.0]))
self.assertEqual(dJdy.shape, (1, ))
assert_almost_equal(dJdy, 2 * xv * av + bv)
net = ComputationalGraphLayer(x * x * a + x * b + c)
out = net.forward(xv)
self.assertEqual(out.shape, (1, ))
assert_almost_equal(out, xv**2 * av + xv * bv + cv)
dJdy = net.backward(np.array([1.0]))
self.assertEqual(dJdy.shape, (1, ))
assert_almost_equal(dJdy, 2 * xv * av + bv)
class VanillaNode(GenericLayer):
def __init__(self,
input_size,
output_size,
memory_size,
Wxh='gaussian',
Whh='gaussian',
Why='gaussian',
bh='zeros',
by='zeros'):
self.memory_size = memory_size
x = Input(['x', 'h'], 'x')
h = Input(['x', 'h'], 'h')
s = Input(['s'], 's')
self.Wxh = MWeight(input_size, memory_size, weights=Wxh)
self.Whh = MWeight(memory_size, memory_size, weights=Whh)
self.bh = VWeight(memory_size, weights=bh)
self.Why = MWeight(memory_size, output_size, weights=Why)
self.by = VWeight(output_size, weights=by)
self.statenet = ComputationalGraphLayer(
Tanh(self.Wxh.dot(x) + self.Whh.dot(h) + self.bh))
self.outputnet = ComputationalGraphLayer(self.Why.dot(s) + self.by)
self.state = np.zeros(memory_size)
self.dJdstate = np.zeros(memory_size)
def forward(self, x_h, update=False):
self.state = self.statenet.forward(x_h)
self.y = self.outputnet.forward(self.state)
return [self.y, self.state]
def backward(self, dJdy_dJdh, optimizer=None):
dJds = self.outputnet.backward(dJdy_dJdh[0], optimizer)
dJdx_dstate = self.statenet.backward(dJds + dJdy_dJdh[1], optimizer)
return dJdx_dstate
def test_variable_dict(self):
xv = np.array([0.5, 0.1, 0.5])
yv = np.array([0.2, 0.4, 0.5])
valin = ['x', 'y']
x = Input(valin, 'x')
y = Input(valin, 'y')
xyv = [xv, yv]
Wxv = np.array([[2.1, 3.1, 2.2], [2.2, 3.2, 4.2], [2.2, 5.2, 4.2]])
Wyv = np.array([[2.1, 2.1, 2.2], [1.6, 1.2, 6.2], [2.1, 3.1, 2.2]])
Wx = MWeight(3, 3, weights=Wxv)
Wy = MWeight(3, 3, weights=Wyv)
net = ComputationalGraphLayer(Sigmoid(Wx.dot(x)) + Tanh(Wy.dot(y)))
netDict = Sequential(VariableDictLayer(valin), net)
out = net.forward(xyv)
self.assertEqual(out.shape, (3, ))
assert_almost_equal(out, sigmoid(Wxv.dot(xv)) + np.tanh(Wyv.dot(yv)))
dJdy = net.backward(np.array([1.0, 1.0, 1.0]))
self.assertEqual(len(dJdy), 2)
for ind, key in enumerate(dJdy):
self.assertEqual(dJdy[ind].shape, xyv[ind].shape)
assert_almost_equal(dJdy[ind],
np.sum(net.numeric_gradient(xyv)[ind], 0))
auxdict = {'x': 0, 'y': 1}
out = netDict.forward({'x': xv, 'y': yv})
self.assertEqual(out.shape, (3, ))
assert_almost_equal(out, sigmoid(Wxv.dot(xv)) + np.tanh(Wyv.dot(yv)))
dJdy = netDict.backward(np.array([1.0, 1.0, 1.0]))
self.assertEqual(len(dJdy), 2)
for key in dJdy:
self.assertEqual(dJdy[key].shape, xyv[auxdict[key]].shape)
assert_almost_equal(
dJdy[key], np.sum(net.numeric_gradient(xyv)[auxdict[key]], 0))
def test_variable_mw_and_w_multi_input(self):
#test one layer W*x+b+W*y+b
list_var = ['x', 'y']
x = Input(list_var, 'x')
y = Input(list_var, 'y')
xv = np.array([1.5, 1.1, 7.5])
yv = np.array([3.5, 3.1, 2.5, 4.3])
xyv = [xv, yv]
Wv1 = np.array([[2.1, 3.1, 2.2], [2.2, 3.2, 4.2]])
Wv2 = np.array([[6.1, 5.1, 2.2, 4.3], [1.2, 1.2, 5.2, 5.1]])
W1 = MWeight(3, 2, weights=Wv1)
W2 = MWeight(4, 2, weights=Wv2)
bv = np.array([1.3, 5.1])
b = VWeight(2, weights=bv)
net = ComputationalGraphLayer(W1.dot(x) + b + W2.dot(y) - b)
out = net.forward(xyv)
self.assertEqual(out.shape, (2, ))
assert_almost_equal(out, Wv1.dot(xv) + bv + Wv2.dot(yv) - bv)
dJdy = net.backward(np.array([1.0, 1.0]))
self.assertEqual(len(dJdy), 2)
gradvett = [np.sum(Wv1, 0), np.sum(Wv2, 0)]
for ind, element in enumerate(dJdy):
self.assertEqual(element.shape, xyv[ind].shape)
assert_almost_equal(dJdy[ind], gradvett[ind])
def test_multi_input(self):
list_var = [
'x', 'y'
] #Here the order is important to understand the order of variable in the input group
x = Input(list_var, 'x')
y = Input(list_var, 'y')
xv = np.array([1.3])
yv = np.array([2.3])
xyv = [xv, yv]
net = ComputationalGraphLayer(y * x + x * x * 3.0 + y * y * 4.3 + 3.1 -
1.2 - x * (y**3 + x * 1.2 + 2) + 1)
out = net.forward(xyv)
self.assertEqual(out.shape, (1, ))
assert_almost_equal(
out, yv * xv + xv * xv * 3.0 + yv * yv * 4.3 + 3.1 - 1.2 - xv *
(yv**3 + xv * 1.2 + 2) + 1)
dJdy = net.backward(np.array([1.0]))
self.assertEqual(len(dJdy), 2)
for element in dJdy:
self.assertEqual(element.shape, (1, ))
assert_almost_equal(
dJdy, [-2 + 3.6 * xv + yv - yv**3, xv + 8.6 * yv - 3 * xv * yv**2])
xv = np.array([1.3, 1.1, 2.3, 1.2])
yv = np.array([2.3, 1.2, 3.1, 2.2])
xyv = [xv, yv]
net = ComputationalGraphLayer(y * x + x * x * 3.0 + y * y * 4.3 + 3.1 -
1 - x * (y**3 + x * 1.2 + 2) + 1)
out = net.forward(xyv)
self.assertEqual(out.shape, (4, ))
assert_almost_equal(
out, yv * xv + xv * xv * 3.0 + yv * yv * 4.3 + 3.1 - 1 - xv *
(yv**3 + xv * 1.2 + 2) + 1)
dJdy = net.backward(np.array([1.0]))
self.assertEqual(len(dJdy), 2)
for element in dJdy:
self.assertEqual(element.shape, (4, ))
assert_almost_equal(
dJdy, [-2 + 3.6 * xv + yv - yv**3, xv + 8.6 * yv - 3 * xv * yv**2])
def test_neuron_one_input(self):
xv = np.array([0.5, 0.1, 0.5])
x = Input(['x'], 'x')
Wv = np.array([[0.1, 0.1, 0.2], [0.5, 0.2, 0.2]])
W = MWeight(3, 2, weights=Wv)
bv = np.array([0.3, 0.1])
b = VWeight(2, weights=bv)
net = ComputationalGraphLayer(Sigmoid(W.dot(x) + b))
out = net.forward(xv)
self.assertEqual(out.shape, (2, ))
check_out = 1.0 / (1.0 + np.exp(-Wv.dot(xv) - bv))
assert_almost_equal(out, check_out)
dJdy = net.backward(np.array([1.0, 1.0]))
self.assertEqual(dJdy.shape, (3, ))
assert_almost_equal(dJdy, np.sum(net.numeric_gradient(xv), 0))
assert_almost_equal(dJdy, (check_out * (1 - check_out)).dot(Wv))
net2 = Sequential(
LinearLayer(3, 2, weights=np.hstack([Wv, bv.reshape(2, 1)])),
SigmoidLayer)
out2 = net2.forward(xv)
assert_almost_equal(out, out2)
dJdy2 = net.backward(np.array([1.0, 1.0]))
assert_almost_equal(dJdy, dJdy2)
def test_variable_mw_and_w_one_input(self):
#test one layer W*x+b
xv = np.array([1.5, 1.1, 7.5])
x = Input(['x'], 'x')
Wv = np.array([[2.1, 3.1, 2.2], [2.2, 3.2, 4.2]])
W = MWeight(3, 2, weights=Wv)
bv = np.array([1.3, 5.1])
b = VWeight(2, weights=bv)
net = ComputationalGraphLayer(W.dot(x) + b)
out = net.forward(xv)
self.assertEqual(out.shape, (2, ))
assert_almost_equal(out, Wv.dot(xv) + bv)
dJdy = net.backward(np.array([1.0, 1.0]))
self.assertEqual(dJdy.shape, (3, ))
assert_almost_equal(dJdy, np.sum(Wv, 0))
def test_variable_matrixweight_one_input(self):
# function W*x vettoriale
xv = np.array([1.3, 1.1, 7.5])
x = Input(['x'], 'x')
Wv = np.array([[2.1, 3.1, 2.2], [2.2, 3.2, 4.2]])
W = MWeight(3, 2, weights=Wv)
net = ComputationalGraphLayer(W.dot(x))
out = net.forward(xv)
self.assertEqual(out.shape, (2, ))
assert_almost_equal(out, Wv.dot(xv))
dJdy = net.backward(np.array([1.0, 1.0]))
self.assertEqual(dJdy.shape, (3, ))
assert_almost_equal(dJdy, np.sum(Wv, 0))
Wv1 = np.array([[2.1, 3.1, 2.2], [2.2, 3.2, 4.2]])
Wv2 = np.array([[2.1, 3.1], [7.4, 2.2], [3.2, 2.2], [1.1, 1.2]])
Wv3 = np.array([[2.1, 3.1, 7.4, 2.2], [3.2, 2.2, 1.1, 1.2],
[2.2, 3.2, 4.2, 7.4]])
W1 = MWeight(3, 2, weights=Wv1)
W2 = MWeight(2, 4, weights=Wv2)
W3 = MWeight(4, 3, weights=Wv3)
net = ComputationalGraphLayer(W3.dot(W2).dot(W1).dot(x))
out = net.forward(xv)
self.assertEqual(out.shape, (3, ))
assert_almost_equal(out, Wv3.dot(Wv2).dot(Wv1).dot(xv))
dJdy = net.backward(np.array([1.0, 1.0, 1.0]))
self.assertEqual(dJdy.shape, (3, ))
assert_almost_equal(dJdy, np.sum(Wv3.dot(Wv2).dot(Wv1), 0))
net = ComputationalGraphLayer(W3.dot(W2.dot(W1.dot(x))))
out = net.forward(xv)
self.assertEqual(out.shape, (3, ))
assert_almost_equal(out, Wv3.dot(Wv2.dot(Wv1.dot(xv))))
dJdy = net.backward(np.array([1.0, 1.0, 1.0]))
self.assertEqual(dJdy.shape, (3, ))
assert_almost_equal(dJdy, np.sum(Wv3.dot(Wv2).dot(Wv1), 0))
def test_concat(self):
xv = np.array([0.5, 0.1, 0.5])
yv = np.array([0.2, 0.4, 0.5, 7.0])
valin = ['x', 'y']
x = Input(valin, 'x')
y = Input(valin, 'y')
xyv = [xv, yv]
Wxv = np.array([[2.1, 3.1, 2.2], [2.2, 3.2, 4.2], [2.2, 5.2, 4.2]])
Wx = MWeight(3, 3, weights=Wxv)
net = ComputationalGraphLayer(Concat([Wx.dot(x), y]))
out = net.forward(xyv)
self.assertEqual(out.shape, (7, ))
dJdy = net.backward(np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]))
self.assertEqual(len(dJdy), 2)
gradvett = [np.sum(Wxv, 0), np.array([1.0, 1.0, 1.0, 1.0])]
for ind, element in enumerate(dJdy):
self.assertEqual(element.shape, xyv[ind].shape)
assert_almost_equal(dJdy[ind], gradvett[ind])
def test_variable_sharedweights(self):
#test one layer W*x+b+W*y+b
list_var = ['x', 'y']
x = Input(list_var, 'x')
y = Input(list_var, 'y')
xv = np.array([1.5, 1.1, 7.5])
yv = np.array([0.2, 0.4, 0.5])
xyv = [xv, yv]
Wv = np.array([[2.1, 3.1, 2.2], [2.2, 3.2, 4.2], [2.2, 5.2, 4.2]])
W = MWeight(3, 3, weights=Wv)
net = ComputationalGraphLayer(W.dot(W.dot(x) + W.dot(y)))
out = net.forward(xyv)
self.assertEqual(out.shape, (3, ))
assert_almost_equal(out, Wv.dot(Wv.dot(xv) + Wv.dot(yv)))
dJdy = net.backward(np.array([1.0, 1.0, 1.0]))
self.assertEqual(len(dJdy), 2)
gradvett = [np.sum(Wv.dot(Wv), 0), np.sum(Wv.dot(Wv), 0)]
for ind, element in enumerate(dJdy):
self.assertEqual(element.shape, xyv[ind].shape)
assert_almost_equal(dJdy[ind], gradvett[ind])