def test_eval():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
data_a = L.Data()
data_b = L.Data()
r_add = data_a + data_b
r_multiply = data_a * data_b
r_subtract = data_a - data_b
datas = {data_a: a, data_b: b}
assert np.allclose(r_add.eval(datas), a + b)
assert np.allclose(r_multiply.eval(datas), np.multiply(a, b))
assert np.allclose(r_subtract.eval(datas), a - b)
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
datas = {data_a: a, data_b: b}
assert np.allclose(r_add.eval(datas), a + b)
assert np.allclose(r_multiply.eval(datas), np.multiply(a, b))
assert np.allclose(r_subtract.eval(datas), a - b)
N, C, H, W = 1, 3, 2, 4
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
datas = {data_a: a, data_b: b}
assert np.allclose(r_add.eval(datas), a + b)
assert np.allclose(r_multiply.eval(datas), np.multiply(a, b))
assert np.allclose(r_subtract.eval(datas), a - b)
def test_data_args():
X = np.random.random((10, 3))
Y = np.random.random((10, 3))
d1 = L.Data([X, Y], "data1")
lx1, ly1 = d1()
d2 = L.Data(X, Y, "data2")
lx2, ly2 = d2()
assert np.allclose(lx1.Y, lx2.Y)
assert np.allclose(ly1.Y, ly2.Y)
def test_fc():
X = np.zeros((4, 2, 1, 1))
X[0, :, 0, 0] = [0., 0.]
X[1, :, 0, 0] = [0., 1.]
X[2, :, 0, 0] = [1., 0.]
X[3, :, 0, 0] = [1., 1.]
Y = np.array([8., 10., 12., 14.]).reshape((-1, 1))
data, label = L.Data([X, Y], "Data")()
fc1 = L.FC(data, "fc1", dim_out=1)
loss = L.MSE(fc1, "MSE", label=label)
fc1.reshape()
fc1.W = np.array([1.0, 3.0]).reshape(fc1.W.shape)
fc1.b = np.array([0.0]).reshape(fc1.b.shape)
net = mobula.Net()
net.set_loss(loss)
net.lr = 0.5
for i in range(30):
net.forward()
net.backward()
print("Iter: %d, Cost: %f" % (i, loss.Y))
# forward one more time, because of the change of weights last backward
net.forward()
target = np.dot(X.reshape((4, 2)), fc1.W.T) + fc1.b
print(target, fc1.Y)
assert np.allclose(fc1.Y, target)
def test_slice():
X = np.arange(60).reshape((5, 3, 2, 2))
l = L.Slice(L.Data(X), "slice", axis=0, slice_points=[1, 2])
Y1, Y2, Y3 = l()
l.reshape()
l.forward()
a1 = Y1.Y.size
a2 = Y2.Y.size
a3 = Y3.Y.size
Y1.dY = np.arange(0, a1).reshape(Y1.Y.shape)
Y2.dY = np.arange(a1, a1 + a2).reshape(Y2.Y.shape)
Y3.dY = np.arange(a1 + a2, a1 + a2 + a3).reshape(Y3.Y.shape)
l.backward()
print("X: ", X.shape, X.ravel())
print("Y1: ", Y1.Y.shape, Y1.Y.ravel())
print("Y2: ", Y2.Y.shape, Y2.Y.ravel())
print("Y3: ", Y3.Y.shape, Y3.Y.ravel())
print("dX: ", l.dX.shape, l.dX.ravel())
y = np.concatenate([Y1.Y.ravel(), Y2.Y.ravel(), Y3.Y.ravel()])
assert (X.ravel() == y).all()
assert (l.dX.ravel() == np.arange(l.dX.size)).all()
def test_fc2():
X = np.random.random((4, 2, 1, 1))
Y1 = np.random.random((4, 10))
Y2 = np.random.random((4, 10))
[x, y1, y2] = L.Data([X, Y1, Y2])
fc1 = L.FC(x, dim_out=10)
fc2 = L.FC(x, dim_out=10)
loss1 = L.MSE(fc1, label=Y1)
loss2 = L.MSE(fc2, label=Y2)
net = mobula.Net()
loss = L.L1Loss(loss1 + loss2)
net.set_loss(loss)
L1Loss = mobula.get_layer("L1Loss")
Add = mobula.get_layer(L1Loss.model.name)
net.lr = 0.5
for i in range(30):
net.forward()
net.backward()
print("Iter: %d, Cost: %f" % (i, loss.Y))
# check forward
t1 = np.dot(X.reshape((4, 2)), fc1.W.T) + fc1.b.T
t2 = np.dot(X.reshape((4, 2)), fc2.W.T) + fc2.b.T
# forward one more time, because of the change of weights last backward
net.forward()
assert np.allclose(fc1.Y, t1)
assert np.allclose(fc2.Y, t2)
def test_add4():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
c = np.random.random((N, C, H, W))
data = L.Data([a, b, c], "data")
[la, lb, lc] = data()
l = la + lb + lc
assert np.allclose(a + b + c, l.eval())
def test_subtract_op_l():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
data = L.Data(a)
l = data - 3
assert type(l) == L.SubtractConstantL
assert np.allclose(a - 3, l.eval())
l.dY = np.random.random(l.Y.shape)
l.backward()
assert np.allclose(l.dX, l.dY)
def test_crop2():
X1 = np.arange(3 * 4 * 5 * 5).reshape((3, 4, 5, 5))
X2 = np.zeros((2, 2, 2, 2))
[x1, x2] = L.Data([X1, X2])
l = L.Crop([x1, x2], offset=(1, 2, 1), axis=1)
assert np.allclose(l.eval(), X1[:, 1:3, 2:4, 1:3])
l.dY = np.random.random(l.Y.shape)
l.backward()
dX = np.zeros(X1.shape)
dX[:, 1:3, 2:4, 1:3] = l.dY
assert np.allclose(l.dX, dX)
def test_subtract_op_r():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
data = L.Data(a)
l = 3 - data
print(type(l))
assert type(l) == L.SubtractConstantR
assert np.allclose(3 - a, l.eval())
l.dY = np.random.random(l.Y.shape)
l.backward()
assert np.allclose(l.dX, -l.dY)
def test_add3():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
c = np.random.random((N, C, H, W))
data = L.Data([a, b, c], "data")
[la, lb, lc] = data()
l = la + lb
l.reshape()
l.forward()
assert np.allclose(a + b, l.Y)
def test_data():
X = np.arange(1, 11).reshape((10, 1))
target = [[1,2,3],[4,5,6],[7,8,9],[2,3,4],[5,6,7],[8,9,10]]
def go_data(data):
data.reshape()
for i in range(50):
data.forward()
assert (data.Y.ravel() == target[i % 6]).all()
data = L.Data(X, "data", batch_size = 3)
go_data(data)
data = L.Data(X, "data", batch_size = 3)()
go_data(data)
data = L.Data(X, "data", batch_size = 3)(0)
go_data(data)
data = L.Data([X, X.copy()], "data", batch_size = 3)(1)
go_data(data)
def test_name():
a = np.arange(10)
b = np.zeros((3, 5))
c = np.arange(1, 11)
test1 = [str(L.Data(a)), str(L.Data([a, b])), str(L.Data(None))]
test1_truth = [
"<Data '/Data' input: (10,) num_output: (1)>",
"<Data '/Data' input: [(10,), (3, 5)] num_output: (2)>",
"<Data '/Data' input: None num_output: (0)>"
]
print(test1)
assert test1 == test1_truth
test2 = [str(L.ReLU(L.Data(a))), str(L.ReLU(L.FC(L.Data(b), dim_out=10)))]
test2_truth = [
"<ReLU '/ReLU' input: /Data:0 num_output: (1)>",
"<ReLU '/ReLU' input: /FC:0 num_output: (1)>"
]
print(test2)
assert test2 == test2_truth
la, lc = L.Data([a, c])
concat = L.Concat([la, lc], axis=0)
test3 = [str(concat)]
test3_truth = [
"<Concat '/Concat' input: [/Data:0,/Data:1] num_output: (1)>"
]
print(test3)
assert test3 == test3_truth
l = L.ReLU(a)
test4 = [str(l)]
test4_truth = ["<ReLU '/ReLU' input: (10,) num_output: (1)>"]
print(test4)
assert test4 == test4_truth
def go_eltwise(op):
a = np.array([1, 0, 6]).astype(np.float)
b = np.array([4, 5, 3]).astype(np.float)
print("a: ", a)
print("b: ", b)
data1 = L.Data(a, "data1")
data2 = L.Data(b, "data1")
coeffs = np.array([-1.0, 1.2])
l = L.Eltwise([data1, data2], "eltwise", op=op, coeffs=coeffs)
l.reshape()
l.forward()
print("Y: ", l.Y)
dY = np.array([7, 8, 9]).astype(np.float)
l.dY = dY
print("dY: ", l.dY)
l.backward()
print("dX: ", l.dX[0], l.dX[1])
c0, c1 = coeffs
if op == L.Eltwise.SUM:
Y = c0 * a + c1 * b
dX0 = c0 * dY
dX1 = c1 * dY
elif op == L.Eltwise.PROD:
Y = a * b * c0 * c1
dX0 = b * dY * c0 * c1
dX1 = a * dY * c0 * c1
elif op == L.Eltwise.MAX:
Y = np.max([c0 * a, c1 * b], 0)
i = np.argmax([c0 * a, c1 * b], 0)
dX0 = np.zeros(a.shape)
dX1 = np.zeros(b.shape)
dX0[i == 0] = dY[i == 0] * c0
dX1[i == 1] = dY[i == 1] * c1
print("Y", l.Y, Y)
assert np.allclose(l.Y, Y)
assert np.allclose(l.dX[0], dX0)
assert np.allclose(l.dX[1], dX1)
def test_subtract():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
[la, lb] = L.Data([a, b])
w = la - lb
w.reshape()
assert w.Y.shape == a.shape
w.forward()
w.dY = np.random.random(w.Y.shape)
w.backward()
assert np.allclose(a - b, w.Y)
assert np.allclose(w.dX[0], w.dY)
assert np.allclose(w.dX[1], -w.dY)
def test_multiply():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
[la, lb] = L.Data([a, b])
w = la * lb
w.reshape()
assert w.Y.shape == a.shape
w.forward()
w.dY = np.random.random(w.Y.shape)
w.backward()
assert np.allclose(np.multiply(a, b), w.Y)
assert np.allclose(w.dX[0], np.multiply(w.dY, w.X[1]))
assert np.allclose(w.dX[1], np.multiply(w.dY, w.X[0]))
def test_sigmoid():
X = ((np.arange(10000) - 5000) / 1000.0).reshape((-1, 1, 1, 1))
data = L.Data(X, "data")
data.reshape()
l = L.Sigmoid(data)
l.reshape()
assert l.Y.shape == X.shape
l.forward()
l.dY = np.random.random(l.Y.shape) * 10
l.backward()
enx = np.exp(-X)
assert np.allclose(l.Y.ravel(), (1.0 / (1.0 + enx)).ravel())
assert np.allclose(l.dX.ravel(), (enx / np.square(1 + enx) * l.dY).ravel())
def test_add2():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
c = np.random.random((N, C, H, W))
data = L.Data([a, b, c], "data")
[la, lb, lc] = data()
l = L.Add([la, lb, lc], "op")
l.reshape()
l.forward()
assert np.allclose(a + b + c, l.Y)
l.dY = np.random.random(l.Y.shape)
l.backward()
for i in range(3):
assert np.allclose(l.dX[i], l.dY)
def test_tanh():
X = ((np.arange(10000) - 5000) / 1000.0).reshape((-1, 1, 1, 1))
data = L.Data(X, "data")
data.reshape()
l = L.Tanh(data)
y = l.eval()
p = np.exp(X)
n = np.exp(-X)
ty = (p - n) / (p + n)
assert np.allclose(y, ty)
l.dY = np.random.random(l.Y.shape)
l.backward()
dX = 1.0 - np.square(p - n) / np.square(p + n)
dX *= l.dY
assert np.allclose(dX, l.dX)
def test_mul():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
data = L.Data(a)
l = data * 3
assert type(l) == L.MultiplyConstant
assert np.allclose(a * 3, l.eval())
l.dY = np.random.random(l.Y.shape)
l.backward()
assert np.allclose(l.dX, 3 * l.dY)
l = 3 * data
assert type(l) == L.MultiplyConstant
assert np.allclose(a * 3, l.eval())
l.dY = np.random.random(l.Y.shape)
l.backward()
assert np.allclose(l.dX, 3 * l.dY)
def test_symbols():
N, C, H, W = 2,3,4,5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
[la, lb] = L.Data([a,b])
eq = M.equal(la, lb)
ne = M.not_equal(la, lb)
ge = M.greater_equal(la, lb)
gt = M.greater(la, lb)
le = M.less_equal(la, lb)
lt = M.less(la, lb)
lops = [eq,ne,ge,gt,le,lt]
ops = [operator.eq, operator.ne, operator.ge, operator.gt, operator.le, operator.lt]
for l, op in zip(lops, ops):
assert l.op == op
def test_add_constant():
N, C, H, W = 2,3,4,5
a = np.random.random((N, C, H, W))
data = L.Data(a)
l = data + 39
assert type(l) == L.AddConstant
assert np.allclose(l.eval(), a + 39)
l.dY = np.random.random(l.Y.shape)
l.backward()
assert np.allclose(l.dX, l.dY)
l = 10 + data
assert type(l) == L.AddConstant
assert np.allclose(l.eval(), a + 10)
l.dY = np.random.random(l.Y.shape)
l.backward()
assert np.allclose(l.dX, l.dY)
def test_list_eval():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
l = L.Data([a, b])
[la, lb] = l
w = la + lb
c = np.random.random((N, C, H, W))
d = np.random.random((N, C, H, W))
datas = {l: [c, d]}
np.allclose(c + d, w.eval(datas))
N, C, H, W = 1, 3, 4, 2
c = np.random.random((N, C, H, W))
d = np.random.random((N, C, H, W))
datas = {l: [c, d]}
np.allclose(c + d, w.eval(datas))
def __init__(self, X, labels):
data, label = L.Data([X, labels], "data", batch_size=100)()
conv1 = L.Conv(data, "conv1", dim_out=20, kernel=5)
pool1 = L.Pool(conv1, "pool1", pool=L.Pool.MAX, kernel=2, stride=2)
conv2 = L.Conv(pool1, "conv2", dim_out=50, kernel=5)
pool2 = L.Pool(conv2, "pool2", pool=L.Pool.MAX, kernel=2, stride=2)
fc3 = L.FC(pool2, "fc3", dim_out=500)
relu3 = L.ReLU(fc3, "relu3")
pred = L.FC(relu3, "pred", dim_out=10)
loss = L.SoftmaxWithLoss(pred, "loss", label=label)
# Net Instance
self.net = mobula.Net()
# Set Loss Layer
self.net.set_loss(loss)
def test_selu():
X = ((np.arange(10000) - 5000) / 1000.0).reshape((-1, 1, 1, 1))
data = L.Data(X, "data")
data.reshape()
l = L.SELU(data)
y = l.eval()
ty = np.zeros(X.shape)
ty[X > 0] = l.scale * X[X > 0]
ty[X <= 0] = l.scale * (l.alpha * np.exp(X[X <= 0]) - l.alpha)
assert np.allclose(y, ty)
l.dY = np.random.random(l.Y.shape)
l.backward()
dX = np.zeros(X.shape)
dX[X > 0] = l.scale
dX[X <= 0] = l.scale * l.alpha * np.exp(X[X <= 0])
dX *= l.dY
assert np.allclose(dX, l.dX)
def test_saver():
filename = "tmp.net"
X = np.random.random((4,2,1,1))
Y = np.random.random((4, 10))
x, y = L.Data([X, Y])
fc = L.FC(x, dim_out = 10)
with M.name_scope("mobula"):
prelu = L.PReLU(fc)
loss = L.MSE(prelu, label = y)
net = M.Net()
net.set_loss(loss)
init_params(fc)
init_params(prelu)
# save mobula
M.save_scope(filename, "mobula")
params_f = clear_params(fc)
params_p = clear_params(prelu)
for p in fc.params + prelu.params:
assert np.isnan(p).all()
M.load_scope(filename)
for p in fc.params:
assert np.isnan(p).all()
for i, p in enumerate(prelu.params):
assert np.allclose(p, params_p[i])
init_params(fc)
init_params(prelu)
# save all
M.save_scope(filename)
params_f = clear_params(fc)
params_p = clear_params(prelu)
for p in fc.params + prelu.params:
assert np.isnan(p).all()
M.load_scope(filename)
for i, p in enumerate(fc.params):
assert np.allclose(p, params_f[i])
for i, p in enumerate(prelu.params):
assert np.allclose(p, params_p[i])
os.remove(filename)
def test_crop():
X1 = np.arange(3 * 4 * 5 * 5).reshape((3, 4, 5, 5))
X2 = np.zeros((3, 4, 3, 3))
[x1, x2] = L.Data([X1, X2], "data")()
l = L.Crop([x1, x2], "crop", offset=1, axis=2)
l.reshape()
l.forward()
target = X1[:, :, 1:4, 1:4]
print(target, l.Y)
assert np.allclose(target, l.Y)
l.dY = np.arange(l.Y.size).reshape(l.Y.shape)
l.backward()
tmp = np.zeros(X1.shape)
tmp[:, :, 1:4, 1:4] = l.dY
assert np.allclose(tmp, l.dX)
l2 = L.Crop([x1, x2], "crop", offset=[1, 1], axis=2)
assert (l.offset == l2.offset).all()
def test_PReLU():
X = ((np.arange(10000) - 5000) / 1000.0).reshape((-1, 1, 1, 1))
data = L.Data(X, "data")
data.reshape()
l = L.PReLU(data)
y = l.eval()
ty = np.zeros(X.shape)
ty[X > 0] = X[X > 0]
ty[X <= 0] = l.a * X[X <= 0]
assert np.allclose(y, ty)
l.dY = np.random.random(l.Y.shape)
l.backward()
dX = np.zeros(X.shape)
dX[X > 0] = 1
dX[X <= 0] = l.a
dX *= l.dY
print(dX, l.dX)
assert np.allclose(dX, l.dX)
def test_contrastive():
A = np.array([[1,2,0], [2,1,0]]).astype(np.float)
B = np.array([[0,1,3], [2,3,1]]).astype(np.float)
diff = A - B # 2 x 3
dist_sq = np.sum(np.square(diff), 1).reshape((2,1)) # 2 x 1
dist = np.sqrt(dist_sq) # 2 x 1
print ("diff", diff)
for i in range(2):
for j in range(2):
sim = np.array([i, j]).reshape((2, 1))
data = L.Data([A,B,sim])
[a,b,s] = data()
l = L.ContrastiveLoss([a,b,s], "ctr", margin = 6.0)
print ("sim", s.Y.ravel())
l.forward()
l.backward()
# margin - d
md = l.margin - dist
# max(md, 0)
ma = np.clip(md, 0, np.inf)
same = (s.Y == 1)
print ("sha", dist_sq.shape, same.shape)
sa = dist_sq * same
sb = np.square(ma) * (1 - same)
print ("lr", sa, sb)
target = np.sum(sa + sb) / 2.0 / 2
print ("t", target, l.Y, target - l.Y)
assert np.allclose(target, l.Y)
left = diff / 2.0
right = -ma / 2.0 * diff / dist
w = left * same + right * (1.0 - same)
print ("dx", l.dX[0], w, l.dX[0] - w)
assert np.allclose(l.dX[0], w)
def test_net_saver():
filename = "tmp.net"
X = np.random.random((4,2,1,1))
Y = np.random.random((4, 10))
x, y = L.Data([X, Y])
x = L.FC(x, dim_out = 10)
with M.name_scope("mobula"):
x = L.PReLU(x)
loss = L.MSE(x, label = y)
net = M.Net()
net.set_loss(loss)
net.lr = 0.01
for i in range(10):
net.forward()
net.backward()
net.save(filename)
# random init layers
lst = M.get_layers("/")
assert len(lst) == 4 # Data, FC, PReLU, MSE
k = 0
rec = []
for l in lst:
for i in range(len(l.params)):
rec.append(l.params[i])
l.params[i][...] = None
k += 1
assert k == 3 # FC.W, FC.b, PReLU.a
for l in lst:
for i in range(len(l.params)):
assert np.isnan(l.params[i]).all()
net.load(filename)
h = 0
for l in lst:
for i in range(len(l.params)):
assert np.allclose(rec[h], l.params[i])
h += 1
os.remove(filename)
def test_compare():
N, C, H, W = 2, 3, 4, 5
a = np.random.random((N, C, H, W))
b = np.random.random((N, C, H, W))
a_copy = a.copy()
[la, lb, lac] = L.Data([a, b, a_copy])
ops = [operator.ge, operator.gt, operator.le, operator.lt]
for op in ops:
l = op(la, lb)
l2 = op(la, lac)
assert np.allclose(l.eval(), op(a, b))
assert np.allclose(l2.eval(), op(a, a_copy))
l.backward()
l2.backward()
assert np.allclose(l.dX[0], np.zeros(l.X[0].shape))
assert np.allclose(l.dX[1], np.zeros(l.X[1].shape))
assert np.allclose(l2.dX[0], np.zeros(l2.X[0].shape))
assert np.allclose(l2.dX[1], np.zeros(l2.X[1].shape))
