def check_backward(self, x, gy, pp): x_mdarray = ideep4py.mdarray(x) gy_mdarray = ideep4py.mdarray(gy) (y_act, ws) = localResponseNormalization.Forward(x_mdarray, pp) gx_act = localResponseNormalization.Backward(x_mdarray, gy_mdarray, ws, pp) gx_act = numpy.array(gx_act, dtype=self.dtype) half_n = self.pp.n // 2 x2 = numpy.square(x) sum_part = x2.copy() for i in six.moves.range(1, half_n + 1): sum_part[:, i:] += x2[:, :-i] sum_part[:, :-i] += x2[:, i:] self.unit_scale = pp.k + pp.alpha * sum_part self.scale = self.unit_scale**-pp.beta self.y = x_mdarray * self.scale summand = self.y * gy / self.unit_scale sum_p = summand.copy() for i in six.moves.range(1, half_n + 1): sum_p[:, i:] += summand[:, :-i] sum_p[:, :-i] += summand[:, i:] gx_expect = gy * self.scale - 2 * pp.alpha * pp.beta * x * sum_p numpy.testing.assert_allclose(gx_expect, gx_act, **self.check_backward_options)
def test_set4(self): x = numpy.array([0, 0, 0, 0], dtype=numpy.float32) mx1 = mdarray(x) mx2 = mdarray(x) mx1.fill(0) mx2.set(mx1) numpy.testing.assert_allclose(mx1, mx2, **self.check_options)
def test_copyto1(self): mx1 = ideep4py.mdarray(self.x1) mx2 = ideep4py.mdarray(self.x2) numpy.copyto(self.x2, self.x1) ideep4py.basic_copyto(mx2, mx1) t = numpy.asarray(mx2) numpy.testing.assert_allclose(t, self.x2, **self.check_options)
def check_backward(self, args, y_grad): x, gamma, beta = args gy = y_grad expander = self.expander inv_m = gamma.dtype.type(1. / (x.size // gamma.size)) expand_dim = False if x.ndim == 2: expand_dim = True x = x[:, :, None, None] gy = gy[:, :, None, None] gamma = gamma[self.expander] beta = numpy.zeros_like(gamma) W = numpy.concatenate((gamma, beta), axis=0).reshape((2, -1)) gx_act, gW = batchNormalization.Backward(ideep4py.mdarray(x), ideep4py.mdarray(gy), ideep4py.mdarray(self.mean), ideep4py.mdarray(self.var), ideep4py.mdarray(W), self.eps) if expand_dim: gx_act = numpy.squeeze(gx_act, axis=(2, 3)) gx_act = numpy.array(gx_act, dtype=self.dtype) self.inv_std = self.var**(-0.5) gbeta = y_grad.sum(axis=self.aggr_axes) x_hat = _x_hat(x, self.mean[expander], self.inv_std[expander]) ggamma = (y_grad * x_hat).sum(axis=self.aggr_axes) gx_expect = (self.gamma * self.inv_std)[expander] * ( y_grad - (x_hat * ggamma[expander] + gbeta[expander]) * inv_m) numpy.testing.assert_allclose(gx_expect, gx_act, **self.check_backward_options)
def check_forward(self, args): x, gamma, beta = args expander = (None, Ellipsis) + (None,) * (x.ndim - self.head_ndim) self.expander = expander self.axis = (0,) + tuple(range(self.head_ndim, x.ndim)) expand_dim = False if x.ndim == 2: expand_dim = True x = x[:, :, None, None] gamma = gamma[expander] beta = beta[expander] y_act, self.mean, self.var, inv_std = batchNormalization.Forward( ideep4py.mdarray(x), ideep4py.mdarray(gamma), ideep4py.mdarray(beta), None, None, self.eps ) if expand_dim: y_act = numpy.squeeze(y_act, axis=(2, 3)) y_act = numpy.array(y_act, dtype=self.dtype) y_expect = _batch_normalization( self.expander, self.gamma, self.beta, self.x, self.mean, self.var) numpy.testing.assert_allclose( y_expect, y_act, **self.check_forward_options)
def run(): src = numpy.arange(3 * 2 * 2 * 2, dtype=numpy.float32) src = src.reshape((3, 2, 2, 2)) src = ideep4py.mdarray(src) gamma = numpy.ones(2, dtype=numpy.float32) beta = numpy.zeros(2, dtype=numpy.float32) w = numpy.concatenate((gamma, beta), axis=0).reshape((2, -1)) w = ideep4py.mdarray(w) eps = 2e-5 print("FWD *****************************") y = batchNormalization.Forward(src, w, None, None, eps) print(y) print(-y[0]) print(-y[1]) print(-y[2]) print("==============") y = batchNormalization.Forward(src, w, None, None, eps) print(y) print(-y[0]) print(-y[1]) print(-y[2]) print("==============") mean = y[1] var = y[2] y = batchNormalization.Forward(src, w, mean, var, eps) print(y) print(-y[0]) print("==============") print("BWD *****************************") diff_dst = numpy.ones(src.shape, dtype=numpy.float32) diff_dst = ideep4py.mdarray(diff_dst) y = batchNormalization.Backward(src, diff_dst, mean, var, w, eps) print(y) print(-y[0]) print(-y[1]) print("==============") y = batchNormalization.Backward(src, diff_dst, mean, var, w, eps) print(y) print(-y[0]) print(-y[1]) print("==============") src = numpy.arange(3 * 2 * 3 * 3, dtype=numpy.float32) src = src.reshape((3, 2, 3, 3)) src = ideep4py.mdarray(src) diff_dst = numpy.ones(src.shape, dtype=numpy.float32) diff_dst = ideep4py.mdarray(diff_dst) y = batchNormalization.Backward(src, diff_dst, mean, var, w, eps) print(y) print(-y[0]) print(-y[1]) print("==============") y = batchNormalization.Backward(src, diff_dst, mean, var, None, eps) print(y) print(-y[0]) print("==============")
def check_backward_data(self, x, W, gy): gx_expect = gy.dot(W).astype(gy.dtype, copy=False) W = ideep4py.mdarray(W) gy = ideep4py.mdarray(gy) gx_act = linear.BackwardData(W, gy) gx_act = numpy.array(gx_act, dtype=self.W_dtype) numpy.testing.assert_allclose(gx_expect, gx_act, **self.check_backward_options)
def test_sum(self): mx1 = ideep4py.mdarray(self.x1) mx2 = ideep4py.mdarray(self.x2) mx3 = ideep4py.mdarray(self.x3) mx4 = ideep4py.mdarray(self.x4) x = self.x1 + self.x2 + self.x3 + self.x4 mx = ideep4py.basic_acc_sum((mx1, mx2, mx3, mx4)) # mx = numpy.asarray(mx) numpy.testing.assert_allclose(mx, x, **self.check_options)
def check_backward_weights(self, x, gy): gW_expect = gy.T.dot(x).astype(self.W_dtype, copy=False) x = ideep4py.mdarray(x) gy = ideep4py.mdarray(gy) gW_act = linear.BackwardWeights(x, gy) gW_act = numpy.array(gW_act, dtype=self.W_dtype) numpy.testing.assert_allclose(gW_expect, gW_act, **self.check_backward_options)
def test_tanhBackward(self): x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype) gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype) gx = gy * (1 - numpy.tanh(x)**2) mx = ideep4py.mdarray(x) mgy = ideep4py.mdarray(gy) mgx = ideep4py._ideep4py.tanh.Backward(mx, mgy) gx1 = numpy.array(mgx) numpy.testing.assert_allclose(gx1, gx, **self.check_options)
def check_forward(self, x, W, b, y_expect): with_bias = True if b is not None else False x = ideep4py.mdarray(x) W = ideep4py.mdarray(W) if with_bias: b = ideep4py.mdarray(b) y_act = linear.Forward(x, W, b) else: y_act = linear.Forward(x, W, None) y_act = numpy.array(y_act, dtype=self.x_dtype) numpy.testing.assert_allclose(y_expect, y_act, **self.check_forward_options)
def run(): print('[1]') x = numpy.ndarray(shape=(256, 3, 224, 224), dtype=numpy.float32, order='C') print('[2]') x = ideep4py.mdarray(x) print('[3]') x = ideep4py.mdarray(x) print('[4]') x = ideep4py.mdarray(x) print('[5]') x = ideep4py.mdarray(x) print('[6]') x = ideep4py.mdarray(x) print('[7]') x = ideep4py.mdarray(x) x1 = numpy.ndarray(shape=(2, 2, 2, 2), dtype=numpy.float32, order='C') x = ideep4py.mdarray(x1) x = x + 1 testing.assert_allclose(x1 + 1, x) x = ideep4py.mdarray(x1) print(x) print("ndims=", x.ndim) print("shape=", x.shape) print("size=", x.size) print("dtype=", x.dtype) print("is_mdarry=", x.is_mdarray) x1 += x x += x x2 = numpy.array(x) testing.assert_allclose(x1, x2) x1 = numpy.ones(shape=(2, 2, 2, 2), dtype=numpy.float32, order='C') x = ideep4py.mdarray(x1) y = x + x1 y2 = numpy.array(y) testing.assert_allclose(y2, x1 + x1) y = x * x1 y2 = numpy.array(y) testing.assert_allclose(y2, x1 * x1) x1 = numpy.random.uniform(-1, 1, (3, 4)).astype(numpy.float32) x = ideep4py.mdarray(x1) z1 = (x1 > 0).astype(x1.dtype) z = (x > 0).astype(x1.dtype) testing.assert_allclose(z, z1)
def check_backward(self, x, gy, pp): # self.shape[2:] h, w = 4, 3 gcol = numpy.tile(gy[:, :, None, None], (1, 1, 3, 3, 1, 1)) gx_expect = col2im_cpu(gcol, 2, 2, 1, 1, h, w) gx_expect /= 3 * 3 gy_mdarray = ideep4py.mdarray(gy) x_mdarray = ideep4py.mdarray(x) gx_act = pooling2D.Backward(x_mdarray, gy_mdarray, None, pp) gx_act = numpy.array(gx_act, dtype=self.dtype) numpy.testing.assert_allclose( gx_expect, gx_act, **self.check_backward_options)
def test_zip(self): mx1 = mdarray(self.x1) mx2 = mdarray(self.x2) a1 = [] a2 = [] b1 = [] b2 = [] for x, y in six.moves.zip(self.x1, self.x2): a1.append(x) a2.append(y) for mx, my in six.moves.zip(mx1, mx2): b1.append(mx) b2.append(my) numpy.testing.assert_allclose(numpy.asarray(a1), numpy.asarray(b1)) numpy.testing.assert_allclose(numpy.asarray(a2), numpy.asarray(b2))
def check_backward(self, x_md, gy): mask, y = dropout.Forward(x_md, self.dropout_ratio) gy_md = ideep4py.mdarray(gy) gx = dropout.Backward(mask, gy_md) gx = numpy.array(gx, dtype=self.dtype) gx_expect = gy * mask numpy.testing.assert_allclose(gx, gx_expect)
def check_forward(self, x, y): mx = ideep4py.mdarray(x) x2 = numpy.array(mx) numpy.testing.assert_allclose(x, x2) my = relu.Forward(mx) y2 = numpy.array(my) numpy.testing.assert_allclose(y, y2)
def test_add(self): x = ideep4py.mdarray(self.x1) y = self.x1 y += x x += x x2 = numpy.array(x) numpy.testing.assert_allclose(x2, y, **self.check_options)
def test_memcpy(self): x1 = numpy.ndarray(shape=self.shape, dtype=self.dtype, order='C') x = ideep4py.mdarray(x1) x2 = numpy.array(x) print("x = ", x1) print("x2 = ", x2) numpy.testing.assert_allclose(x, x2, **self.check_options)
def test_attriMdarray(self): x = ideep4py.mdarray(self.x1) self.assertEqual(x.ndim, self.x1.ndim) self.assertEqual(x.shape, self.x1.shape) self.assertEqual(x.size, self.x1.size) self.assertEqual(x.dtype, self.x1.dtype) self.assertTrue(x.is_mdarray)
def test_set1(self): x = numpy.array([1, 1, 1], dtype=numpy.float32) mx = mdarray(x) numpy.testing.assert_allclose(mx, x, **self.check_options) x = numpy.array([1, 2, 1], dtype=numpy.float32) mx.set(x) numpy.testing.assert_allclose(mx, x, **self.check_options)
def test_set2(self): x = numpy.arange(24, dtype=numpy.float32) mx = mdarray(x) numpy.testing.assert_allclose(mx, x, **self.check_options) x.fill(1) mx.set(x) numpy.testing.assert_allclose(mx, x, **self.check_options)
def test_sum7(self): x = numpy.random.rand(256, 1000) x = x.astype(numpy.float32) mx = mdarray(x) ms = mx.sum((1)) ns = x.sum((1)) numpy.testing.assert_allclose(ms, ns, **self.check_options)
def test_sum4(self): x = numpy.random.rand(256, 384, 13, 13) x = x.astype(numpy.float32) y = numpy.maximum(x, 0, dtype=numpy.float32) mx = mdarray(x) my = relu.Forward(mx) numpy.testing.assert_allclose(my.sum((0, 2, 3)), y.sum((0, 2, 3)), **self.check_options) numpy.testing.assert_allclose(my.sum((1, 2, 3)), y.sum((1, 2, 3)), **self.check_options) numpy.testing.assert_allclose(my.sum((0, 1, 2)), y.sum((0, 1, 2)), **self.check_options) numpy.testing.assert_allclose(my.sum((0, 2)), y.sum((0, 2)), **self.check_options) numpy.testing.assert_allclose(my.sum((1, 3)), y.sum((1, 3)), **self.check_options) numpy.testing.assert_allclose(my.sum((1, 2)), y.sum((1, 2)), **self.check_options) numpy.testing.assert_allclose(my.sum((0)), y.sum((0)), **self.check_options) numpy.testing.assert_allclose(my.sum((3)), y.sum((3)), **self.check_options) numpy.testing.assert_allclose(my.sum((2)), y.sum((2)), **self.check_options) numpy.testing.assert_allclose(my.sum((0, 2, 3)), y.sum((0, 2, 3)), **self.check_options) numpy.testing.assert_allclose(my.sum((0, 2, 3), keepdims=True), y.sum((0, 2, 3), keepdims=True), **self.check_options)
def test_value_change(self): # value change x1 = numpy.ndarray(shape=(2, 2, 2, 2), dtype=numpy.float32, order='C') x = ideep4py.mdarray(x1) y = x.reshape(len(x), -1) x[0, 0, 0, 0] = 3.333 self.assertEqual(x[0, 0, 0, 0], y[0, 0]) x[0, 0, 0, 0] = 4.4444 self.assertEqual(x[0, 0, 0, 0], y[0, 0])
def test_set3(self): x = numpy.random.rand(10, 10, 10, 10) x = x.astype(numpy.float32) mx = mdarray(x) numpy.testing.assert_allclose(mx, x, **self.check_options) x = numpy.random.rand(10, 10, 10, 10) x = x.astype(numpy.float32) mx.set(x) numpy.testing.assert_allclose(mx, x, **self.check_options)
def check_backward(self, xs_data, y_data, axis): xs = tuple(x_data for x_data in xs_data) xs_mdarray = mdarrayVector() for yi in xs: if isinstance(yi, numpy.ndarray): if yi.flags.contiguous is False: yi = numpy.ascontiguousarray(yi) yi = ideep4py.mdarray(numpy.ascontiguousarray(yi)) xs_mdarray.push_back(yi) y_data = ideep4py.mdarray(y_data) offsets = intVector() # FIXME for i in self.section: offsets.push_back(i) x_act_mdarray = concat.Backward(y_data, offsets, self.axis) i = 0 for x in xs: x_act = numpy.array(x_act_mdarray[i], dtype=self.dtype) numpy.testing.assert_allclose(x, x_act, atol=0, rtol=0) i = i + 1
def check_forward(self, x, pp): x_mdarray = ideep4py.mdarray(x) (y_act,) = pooling2D.Forward(x_mdarray, pp) y_act = numpy.array(y_act, dtype=self.dtype) for k in six.moves.range(self.bs): for c in six.moves.range(self.channel): x = self.x[k, c] expect = numpy.array([ [x[0:2, 0:2].sum(), x[0:2, 1:3].sum()], [x[1:4, 0:2].sum(), x[1:4, 1:3].sum()]]) / 9 numpy.testing.assert_allclose( expect, y_act[k, c], **self.check_forward_options)
def check_forward(self, xs_data, y_data, axis): xs = tuple(x_data for x_data in xs_data) xs_mdarray = mdarrayVector() for yi in xs: if isinstance(yi, numpy.ndarray): if yi.flags.contiguous is False: yi = numpy.ascontiguousarray(yi) yi = ideep4py.mdarray(numpy.ascontiguousarray(yi)) xs_mdarray.push_back(yi) y_act = concat.Forward(xs_mdarray, self.axis) y_act = numpy.array(y_act, dtype=self.dtype) numpy.testing.assert_allclose(y_data, y_act, atol=0, rtol=0)
def setUp(self): self.x_shape = (self.bs, self.channel, 224, 224) self.w_shape = (self.channel, self.channel, 3, 3) self.b_shape = self.channel self.x = numpy.random.uniform(-1, 1, self.x_shape).astype(self.dtype) self.x = ideep4py.mdarray(self.x) self.w = numpy.random.uniform(-1, 1, self.w_shape).astype(self.dtype) self.w = ideep4py.mdarray(self.w) self.b = numpy.random.uniform(-1, 1, self.b_shape).astype(self.dtype) self.b = ideep4py.mdarray(self.b) self.cp = convolution2DParam(self.x_shape, 1, 1, 1, 1, 1, 1, 1, 1) stride = 1 pad = 1 dilate = 1 self.sy, self.sx = stride, stride self.ph, self.pw = pad, pad self.n = self.x_shape[0] self.outc = self.w_shape[0] self.outh = self.x_shape[2] self.outw = self.x_shape[3] self.cover_all = self.cover_all self.dy, self.dx = dilate, dilate self.gy = numpy.random.uniform( -1, 1, (self.n, self.outc, self.outh, self.outw)).astype(self.dtype) self.gy = ideep4py.mdarray(self.gy) self.check_forward_options = {'atol': 1e-3, 'rtol': 1e-2} self.check_backward_options = {'atol': 1e-3, 'rtol': 1e-2}
def check_forward(self, x, pp): x_mdarray = ideep4py.mdarray(x) (y_act, ws) = localResponseNormalization.Forward(x_mdarray, pp) y_act = numpy.array(y_act, dtype=self.dtype) y_expect = numpy.zeros_like(self.x) for n, c, h, w in numpy.ndindex(self.x.shape): s = 0 for i in six.moves.range(max(0, c - 2), min(7, c + 2)): s += self.x[n, i, h, w]**2 denom = (2 + 1e-4 * s)**.75 y_expect[n, c, h, w] = self.x[n, c, h, w] / denom numpy.testing.assert_allclose(y_expect, y_act, **self.check_forward_options)