def check_backward(self, x, gy, pp):
x_mdarray = ideep4py.mdarray(x)
gy_mdarray = ideep4py.mdarray(gy)
(y_act, ws) = localResponseNormalization.Forward(x_mdarray, pp)
gx_act = localResponseNormalization.Backward(x_mdarray, gy_mdarray, ws,
pp)
gx_act = numpy.array(gx_act, dtype=self.dtype)
half_n = self.pp.n // 2
x2 = numpy.square(x)
sum_part = x2.copy()
for i in six.moves.range(1, half_n + 1):
sum_part[:, i:] += x2[:, :-i]
sum_part[:, :-i] += x2[:, i:]
self.unit_scale = pp.k + pp.alpha * sum_part
self.scale = self.unit_scale**-pp.beta
self.y = x_mdarray * self.scale
summand = self.y * gy / self.unit_scale
sum_p = summand.copy()
for i in six.moves.range(1, half_n + 1):
sum_p[:, i:] += summand[:, :-i]
sum_p[:, :-i] += summand[:, i:]
gx_expect = gy * self.scale - 2 * pp.alpha * pp.beta * x * sum_p
numpy.testing.assert_allclose(gx_expect, gx_act,
**self.check_backward_options)
def test_set4(self):
x = numpy.array([0, 0, 0, 0], dtype=numpy.float32)
mx1 = mdarray(x)
mx2 = mdarray(x)
mx1.fill(0)
mx2.set(mx1)
numpy.testing.assert_allclose(mx1, mx2, **self.check_options)
def test_copyto1(self):
mx1 = ideep4py.mdarray(self.x1)
mx2 = ideep4py.mdarray(self.x2)
numpy.copyto(self.x2, self.x1)
ideep4py.basic_copyto(mx2, mx1)
t = numpy.asarray(mx2)
numpy.testing.assert_allclose(t, self.x2, **self.check_options)
def check_backward(self, args, y_grad):
x, gamma, beta = args
gy = y_grad
expander = self.expander
inv_m = gamma.dtype.type(1. / (x.size // gamma.size))
expand_dim = False
if x.ndim == 2:
expand_dim = True
x = x[:, :, None, None]
gy = gy[:, :, None, None]
gamma = gamma[self.expander]
beta = numpy.zeros_like(gamma)
W = numpy.concatenate((gamma, beta), axis=0).reshape((2, -1))
gx_act, gW = batchNormalization.Backward(ideep4py.mdarray(x),
ideep4py.mdarray(gy),
ideep4py.mdarray(self.mean),
ideep4py.mdarray(self.var),
ideep4py.mdarray(W), self.eps)
if expand_dim:
gx_act = numpy.squeeze(gx_act, axis=(2, 3))
gx_act = numpy.array(gx_act, dtype=self.dtype)
self.inv_std = self.var**(-0.5)
gbeta = y_grad.sum(axis=self.aggr_axes)
x_hat = _x_hat(x, self.mean[expander], self.inv_std[expander])
ggamma = (y_grad * x_hat).sum(axis=self.aggr_axes)
gx_expect = (self.gamma * self.inv_std)[expander] * (
y_grad - (x_hat * ggamma[expander] + gbeta[expander]) * inv_m)
numpy.testing.assert_allclose(gx_expect, gx_act,
**self.check_backward_options)
def check_forward(self, args):
x, gamma, beta = args
expander = (None, Ellipsis) + (None,) * (x.ndim - self.head_ndim)
self.expander = expander
self.axis = (0,) + tuple(range(self.head_ndim, x.ndim))
expand_dim = False
if x.ndim == 2:
expand_dim = True
x = x[:, :, None, None]
gamma = gamma[expander]
beta = beta[expander]
y_act, self.mean, self.var, inv_std = batchNormalization.Forward(
ideep4py.mdarray(x),
ideep4py.mdarray(gamma),
ideep4py.mdarray(beta),
None,
None,
self.eps
)
if expand_dim:
y_act = numpy.squeeze(y_act, axis=(2, 3))
y_act = numpy.array(y_act, dtype=self.dtype)
y_expect = _batch_normalization(
self.expander, self.gamma, self.beta, self.x, self.mean, self.var)
numpy.testing.assert_allclose(
y_expect, y_act, **self.check_forward_options)
def run():
src = numpy.arange(3 * 2 * 2 * 2, dtype=numpy.float32)
src = src.reshape((3, 2, 2, 2))
src = ideep4py.mdarray(src)
gamma = numpy.ones(2, dtype=numpy.float32)
beta = numpy.zeros(2, dtype=numpy.float32)
w = numpy.concatenate((gamma, beta), axis=0).reshape((2, -1))
w = ideep4py.mdarray(w)
eps = 2e-5
print("FWD *****************************")
y = batchNormalization.Forward(src, w, None, None, eps)
print(y)
print(-y[0])
print(-y[1])
print(-y[2])
print("==============")
y = batchNormalization.Forward(src, w, None, None, eps)
print(y)
print(-y[0])
print(-y[1])
print(-y[2])
print("==============")
mean = y[1]
var = y[2]
y = batchNormalization.Forward(src, w, mean, var, eps)
print(y)
print(-y[0])
print("==============")
print("BWD *****************************")
diff_dst = numpy.ones(src.shape, dtype=numpy.float32)
diff_dst = ideep4py.mdarray(diff_dst)
y = batchNormalization.Backward(src, diff_dst, mean, var, w, eps)
print(y)
print(-y[0])
print(-y[1])
print("==============")
y = batchNormalization.Backward(src, diff_dst, mean, var, w, eps)
print(y)
print(-y[0])
print(-y[1])
print("==============")
src = numpy.arange(3 * 2 * 3 * 3, dtype=numpy.float32)
src = src.reshape((3, 2, 3, 3))
src = ideep4py.mdarray(src)
diff_dst = numpy.ones(src.shape, dtype=numpy.float32)
diff_dst = ideep4py.mdarray(diff_dst)
y = batchNormalization.Backward(src, diff_dst, mean, var, w, eps)
print(y)
print(-y[0])
print(-y[1])
print("==============")
y = batchNormalization.Backward(src, diff_dst, mean, var, None, eps)
print(y)
print(-y[0])
print("==============")
def check_backward_data(self, x, W, gy):
gx_expect = gy.dot(W).astype(gy.dtype, copy=False)
W = ideep4py.mdarray(W)
gy = ideep4py.mdarray(gy)
gx_act = linear.BackwardData(W, gy)
gx_act = numpy.array(gx_act, dtype=self.W_dtype)
numpy.testing.assert_allclose(gx_expect, gx_act,
**self.check_backward_options)
def test_sum(self):
mx1 = ideep4py.mdarray(self.x1)
mx2 = ideep4py.mdarray(self.x2)
mx3 = ideep4py.mdarray(self.x3)
mx4 = ideep4py.mdarray(self.x4)
x = self.x1 + self.x2 + self.x3 + self.x4
mx = ideep4py.basic_acc_sum((mx1, mx2, mx3, mx4))
# mx = numpy.asarray(mx)
numpy.testing.assert_allclose(mx, x, **self.check_options)
def check_backward_weights(self, x, gy):
gW_expect = gy.T.dot(x).astype(self.W_dtype, copy=False)
x = ideep4py.mdarray(x)
gy = ideep4py.mdarray(gy)
gW_act = linear.BackwardWeights(x, gy)
gW_act = numpy.array(gW_act, dtype=self.W_dtype)
numpy.testing.assert_allclose(gW_expect, gW_act,
**self.check_backward_options)
def test_tanhBackward(self):
x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
gx = gy * (1 - numpy.tanh(x)**2)
mx = ideep4py.mdarray(x)
mgy = ideep4py.mdarray(gy)
mgx = ideep4py._ideep4py.tanh.Backward(mx, mgy)
gx1 = numpy.array(mgx)
numpy.testing.assert_allclose(gx1, gx, **self.check_options)
def check_forward(self, x, W, b, y_expect):
with_bias = True if b is not None else False
x = ideep4py.mdarray(x)
W = ideep4py.mdarray(W)
if with_bias:
b = ideep4py.mdarray(b)
y_act = linear.Forward(x, W, b)
else:
y_act = linear.Forward(x, W, None)
y_act = numpy.array(y_act, dtype=self.x_dtype)
numpy.testing.assert_allclose(y_expect, y_act,
**self.check_forward_options)
def run():
print('[1]')
x = numpy.ndarray(shape=(256, 3, 224, 224), dtype=numpy.float32, order='C')
print('[2]')
x = ideep4py.mdarray(x)
print('[3]')
x = ideep4py.mdarray(x)
print('[4]')
x = ideep4py.mdarray(x)
print('[5]')
x = ideep4py.mdarray(x)
print('[6]')
x = ideep4py.mdarray(x)
print('[7]')
x = ideep4py.mdarray(x)
x1 = numpy.ndarray(shape=(2, 2, 2, 2), dtype=numpy.float32, order='C')
x = ideep4py.mdarray(x1)
x = x + 1
testing.assert_allclose(x1 + 1, x)
x = ideep4py.mdarray(x1)
print(x)
print("ndims=", x.ndim)
print("shape=", x.shape)
print("size=", x.size)
print("dtype=", x.dtype)
print("is_mdarry=", x.is_mdarray)
x1 += x
x += x
x2 = numpy.array(x)
testing.assert_allclose(x1, x2)
x1 = numpy.ones(shape=(2, 2, 2, 2), dtype=numpy.float32, order='C')
x = ideep4py.mdarray(x1)
y = x + x1
y2 = numpy.array(y)
testing.assert_allclose(y2, x1 + x1)
y = x * x1
y2 = numpy.array(y)
testing.assert_allclose(y2, x1 * x1)
x1 = numpy.random.uniform(-1, 1, (3, 4)).astype(numpy.float32)
x = ideep4py.mdarray(x1)
z1 = (x1 > 0).astype(x1.dtype)
z = (x > 0).astype(x1.dtype)
testing.assert_allclose(z, z1)
def check_backward(self, x, gy, pp):
# self.shape[2:]
h, w = 4, 3
gcol = numpy.tile(gy[:, :, None, None],
(1, 1, 3, 3, 1, 1))
gx_expect = col2im_cpu(gcol, 2, 2, 1, 1, h, w)
gx_expect /= 3 * 3
gy_mdarray = ideep4py.mdarray(gy)
x_mdarray = ideep4py.mdarray(x)
gx_act = pooling2D.Backward(x_mdarray, gy_mdarray, None, pp)
gx_act = numpy.array(gx_act, dtype=self.dtype)
numpy.testing.assert_allclose(
gx_expect, gx_act, **self.check_backward_options)
def test_zip(self):
mx1 = mdarray(self.x1)
mx2 = mdarray(self.x2)
a1 = []
a2 = []
b1 = []
b2 = []
for x, y in six.moves.zip(self.x1, self.x2):
a1.append(x)
a2.append(y)
for mx, my in six.moves.zip(mx1, mx2):
b1.append(mx)
b2.append(my)
numpy.testing.assert_allclose(numpy.asarray(a1), numpy.asarray(b1))
numpy.testing.assert_allclose(numpy.asarray(a2), numpy.asarray(b2))
def check_backward(self, x_md, gy):
mask, y = dropout.Forward(x_md, self.dropout_ratio)
gy_md = ideep4py.mdarray(gy)
gx = dropout.Backward(mask, gy_md)
gx = numpy.array(gx, dtype=self.dtype)
gx_expect = gy * mask
numpy.testing.assert_allclose(gx, gx_expect)
def check_forward(self, x, y):
mx = ideep4py.mdarray(x)
x2 = numpy.array(mx)
numpy.testing.assert_allclose(x, x2)
my = relu.Forward(mx)
y2 = numpy.array(my)
numpy.testing.assert_allclose(y, y2)
def test_add(self):
x = ideep4py.mdarray(self.x1)
y = self.x1
y += x
x += x
x2 = numpy.array(x)
numpy.testing.assert_allclose(x2, y, **self.check_options)
def test_memcpy(self):
x1 = numpy.ndarray(shape=self.shape, dtype=self.dtype, order='C')
x = ideep4py.mdarray(x1)
x2 = numpy.array(x)
print("x = ", x1)
print("x2 = ", x2)
numpy.testing.assert_allclose(x, x2, **self.check_options)
def test_attriMdarray(self):
x = ideep4py.mdarray(self.x1)
self.assertEqual(x.ndim, self.x1.ndim)
self.assertEqual(x.shape, self.x1.shape)
self.assertEqual(x.size, self.x1.size)
self.assertEqual(x.dtype, self.x1.dtype)
self.assertTrue(x.is_mdarray)
def test_set1(self):
x = numpy.array([1, 1, 1], dtype=numpy.float32)
mx = mdarray(x)
numpy.testing.assert_allclose(mx, x, **self.check_options)
x = numpy.array([1, 2, 1], dtype=numpy.float32)
mx.set(x)
numpy.testing.assert_allclose(mx, x, **self.check_options)
def test_set2(self):
x = numpy.arange(24, dtype=numpy.float32)
mx = mdarray(x)
numpy.testing.assert_allclose(mx, x, **self.check_options)
x.fill(1)
mx.set(x)
numpy.testing.assert_allclose(mx, x, **self.check_options)
def test_sum7(self):
x = numpy.random.rand(256, 1000)
x = x.astype(numpy.float32)
mx = mdarray(x)
ms = mx.sum((1))
ns = x.sum((1))
numpy.testing.assert_allclose(ms, ns, **self.check_options)
def test_sum4(self):
x = numpy.random.rand(256, 384, 13, 13)
x = x.astype(numpy.float32)
y = numpy.maximum(x, 0, dtype=numpy.float32)
mx = mdarray(x)
my = relu.Forward(mx)
numpy.testing.assert_allclose(my.sum((0, 2, 3)), y.sum((0, 2, 3)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((1, 2, 3)), y.sum((1, 2, 3)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((0, 1, 2)), y.sum((0, 1, 2)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((0, 2)), y.sum((0, 2)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((1, 3)), y.sum((1, 3)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((1, 2)), y.sum((1, 2)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((0)), y.sum((0)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((3)), y.sum((3)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((2)), y.sum((2)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((0, 2, 3)), y.sum((0, 2, 3)),
**self.check_options)
numpy.testing.assert_allclose(my.sum((0, 2, 3), keepdims=True),
y.sum((0, 2, 3), keepdims=True),
**self.check_options)
def test_value_change(self):
# value change
x1 = numpy.ndarray(shape=(2, 2, 2, 2), dtype=numpy.float32, order='C')
x = ideep4py.mdarray(x1)
y = x.reshape(len(x), -1)
x[0, 0, 0, 0] = 3.333
self.assertEqual(x[0, 0, 0, 0], y[0, 0])
x[0, 0, 0, 0] = 4.4444
self.assertEqual(x[0, 0, 0, 0], y[0, 0])
def test_set3(self):
x = numpy.random.rand(10, 10, 10, 10)
x = x.astype(numpy.float32)
mx = mdarray(x)
numpy.testing.assert_allclose(mx, x, **self.check_options)
x = numpy.random.rand(10, 10, 10, 10)
x = x.astype(numpy.float32)
mx.set(x)
numpy.testing.assert_allclose(mx, x, **self.check_options)
def check_backward(self, xs_data, y_data, axis):
xs = tuple(x_data for x_data in xs_data)
xs_mdarray = mdarrayVector()
for yi in xs:
if isinstance(yi, numpy.ndarray):
if yi.flags.contiguous is False:
yi = numpy.ascontiguousarray(yi)
yi = ideep4py.mdarray(numpy.ascontiguousarray(yi))
xs_mdarray.push_back(yi)
y_data = ideep4py.mdarray(y_data)
offsets = intVector()
# FIXME
for i in self.section:
offsets.push_back(i)
x_act_mdarray = concat.Backward(y_data, offsets, self.axis)
i = 0
for x in xs:
x_act = numpy.array(x_act_mdarray[i], dtype=self.dtype)
numpy.testing.assert_allclose(x, x_act, atol=0, rtol=0)
i = i + 1
def check_forward(self, x, pp):
x_mdarray = ideep4py.mdarray(x)
(y_act,) = pooling2D.Forward(x_mdarray, pp)
y_act = numpy.array(y_act, dtype=self.dtype)
for k in six.moves.range(self.bs):
for c in six.moves.range(self.channel):
x = self.x[k, c]
expect = numpy.array([
[x[0:2, 0:2].sum(), x[0:2, 1:3].sum()],
[x[1:4, 0:2].sum(), x[1:4, 1:3].sum()]]) / 9
numpy.testing.assert_allclose(
expect, y_act[k, c], **self.check_forward_options)
def check_forward(self, xs_data, y_data, axis):
xs = tuple(x_data for x_data in xs_data)
xs_mdarray = mdarrayVector()
for yi in xs:
if isinstance(yi, numpy.ndarray):
if yi.flags.contiguous is False:
yi = numpy.ascontiguousarray(yi)
yi = ideep4py.mdarray(numpy.ascontiguousarray(yi))
xs_mdarray.push_back(yi)
y_act = concat.Forward(xs_mdarray, self.axis)
y_act = numpy.array(y_act, dtype=self.dtype)
numpy.testing.assert_allclose(y_data, y_act, atol=0, rtol=0)
def setUp(self):
self.x_shape = (self.bs, self.channel, 224, 224)
self.w_shape = (self.channel, self.channel, 3, 3)
self.b_shape = self.channel
self.x = numpy.random.uniform(-1, 1, self.x_shape).astype(self.dtype)
self.x = ideep4py.mdarray(self.x)
self.w = numpy.random.uniform(-1, 1, self.w_shape).astype(self.dtype)
self.w = ideep4py.mdarray(self.w)
self.b = numpy.random.uniform(-1, 1, self.b_shape).astype(self.dtype)
self.b = ideep4py.mdarray(self.b)
self.cp = convolution2DParam(self.x_shape,
1, 1,
1, 1,
1, 1,
1, 1)
stride = 1
pad = 1
dilate = 1
self.sy, self.sx = stride, stride
self.ph, self.pw = pad, pad
self.n = self.x_shape[0]
self.outc = self.w_shape[0]
self.outh = self.x_shape[2]
self.outw = self.x_shape[3]
self.cover_all = self.cover_all
self.dy, self.dx = dilate, dilate
self.gy = numpy.random.uniform(
-1, 1,
(self.n, self.outc, self.outh, self.outw)).astype(self.dtype)
self.gy = ideep4py.mdarray(self.gy)
self.check_forward_options = {'atol': 1e-3, 'rtol': 1e-2}
self.check_backward_options = {'atol': 1e-3, 'rtol': 1e-2}
def check_forward(self, x, pp):
x_mdarray = ideep4py.mdarray(x)
(y_act, ws) = localResponseNormalization.Forward(x_mdarray, pp)
y_act = numpy.array(y_act, dtype=self.dtype)
y_expect = numpy.zeros_like(self.x)
for n, c, h, w in numpy.ndindex(self.x.shape):
s = 0
for i in six.moves.range(max(0, c - 2), min(7, c + 2)):
s += self.x[n, i, h, w]**2
denom = (2 + 1e-4 * s)**.75
y_expect[n, c, h, w] = self.x[n, c, h, w] / denom
numpy.testing.assert_allclose(y_expect, y_act,
**self.check_forward_options)
