def test_r_div_scalar_forward_backward(seed, val, ctx, func_name): from nbla_test_utils import function_tester, cap_ignore_region rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 3, (-0.5, 0.5))] function_tester(rng, F.r_div_scalar, lambda x, y: y / x, inputs, func_args=[val], dstep=1e-4, atol_b=1e-1, ctx=ctx, func_name=func_name)
def test_crelu_forward_backward(seed, axis, ctx, func_name): from nbla_test_utils import cap_ignore_region, function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3))] function_tester(rng, F.crelu, ref_crelu, inputs, func_args=[axis], ctx=ctx, func_name=func_name, atol_b=1e-2)
def test_abs_forward_backward(seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3))] function_tester(rng, F.abs, np.abs, inputs, ctx=ctx, func_name=func_name)
def test_abs_forward_backward(seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, function_tester rng = np.random.RandomState(seed) inputs = [] for _ in range(2): inputs.append( cap_ignore_region( rng.randn(2, 3,).astype(np.float32) * 2, (-1e-3, 1e-3))) function_tester(rng, F.absolute_error, lambda x, y: np.abs(x - y), inputs, ctx=ctx, func_name=func_name, atol_b=1e-2) # NOTE if atol_b=1e-3, then a numerical error occurs.
def test_r_div_scalar_forward_backward(seed, val, ctx, func_name): from nbla_test_utils import function_tester, cap_ignore_region rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 3, (-0.5, 0.5)) ] function_tester(rng, F.r_div_scalar, lambda x, y: y / x, inputs, func_args=[val], dstep=1e-4, atol_b=1e-1, ctx=ctx, func_name=func_name)
def test_floor_forward_backward(seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, \ function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3))] function_tester(rng, F.floor, ref_floor, inputs, atol_b=1e-3, backward=[True], ctx=ctx, func_name=func_name, ref_grad=ref_grad_floor)
def test_leaky_relu_double_backward(seed, ctx, func_name, alpha, inplace): from nbla_test_utils import cap_ignore_region, backward_function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3)) ] backward_function_tester(rng, F.leaky_relu, inputs=inputs, func_args=[alpha, inplace], func_kwargs={}, atol_accum=1e-3, dstep=1e-3, backward_b=[True, False], ctx=ctx)
def test_crelu_double_backward(seed, axis, ctx, func_name): from nbla_test_utils import cap_ignore_region, backward_function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3)) ] backward_function_tester(rng, F.crelu, None, inputs, func_args=[axis], ctx=ctx, func_name=func_name, atol_b=1e-3, atol_accum=1e-3)
def test_r_sub_scalar_double_backward(seed, val, ctx, func_name): from nbla_test_utils import backward_function_tester, cap_ignore_region rng = np.random.RandomState(seed) inputs = [ cap_ignore_region(rng.randn(2, 3).astype(np.float32) * 3, (-0.5, 0.5)) ] backward_function_tester(rng, F.r_sub_scalar, None, inputs=inputs, func_args=[val], func_kwargs={}, atol_b=1e-3, atol_accum=1e-3, dstep=1e-3, ctx=ctx, func_name=None, disable_half_test=False)
def test_norm_normalization_forward_backward(eps, axis, p, shape, seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, function_tester from sys import platform if platform == "darwin": pytest.skip("NormNormalization is not supported in macOS.") rng = np.random.RandomState(seed) inputs = [cap_ignore_region( rng.randn(*shape).astype(np.float32) * 2, (-1e-3, 1e-3))] func_args = [p, axis, eps] function_tester(rng, F.norm_normalization, ref_norm_normalization, inputs, ctx=ctx, func_name=func_name, func_args=func_args, backward=[ True], disable_half_test=False, # The backward test on macOS doesn't pass with this tolerance. # Does Eigen library used in CPU computation backend produce # the different results on different platforms? # atol_b=3e-3, atol_b=1e-2, atol_accum=1e-2)
def test_norm_normalization_double_backward(eps, axis, p, shape, seed, ctx, func_name): from sys import platform if platform == "darwin": pytest.skip("NormNormalization is not supported in macOS.") from nbla_test_utils import cap_ignore_region, backward_function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(*shape).astype(np.float32) * 2, (-1e-3, 1e-3)) ] func_args = [p, axis, eps] backward_function_tester(rng, F.norm_normalization, inputs=inputs, func_args=func_args, ctx=ctx)
def test_hard_tanh_double_backward(seed, ctx, func_name): from nbla_test_utils import backward_function_tester, cap_ignore_region rng = np.random.RandomState(seed) inputs = [ cap_ignore_region(rng.randn(2, 3).astype(np.float32), (-0.9, 0.9)) ] backward_function_tester(rng, F.hard_tanh, None, inputs=inputs, func_args=[], func_kwargs={}, atol_b=1e-3, atol_accum=1e-3, dstep=1e-3, ctx=ctx, func_name=None, disable_half_test=True)
def test_relu_double_backward(seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, backward_function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3)) ] backward_function_tester(rng, F.relu, None, inputs=inputs, func_args=[], func_kwargs={}, atol_b=1e-3, atol_accum=1e-3, dstep=1e-3, ctx=ctx, func_name=None, disable_half_test=False)
def test_dropout_forward_backward(p, seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3)) ] # Ensure there is no zero. i = nn.Variable(inputs[0].shape, need_grad=True) i.d = inputs[0] # NNabla forward with nn.context_scope(ctx), nn.auto_forward(): o = F.dropout(i, p) scale = 1. / (1. - p) mask = o.d != 0 assert_allclose(o.d, i.d * mask * scale) assert o.parent.name == func_name # NNabla backward orig_grad = rng.randn(*i.shape).astype(i.data.dtype) i.g[...] = orig_grad o_grad = rng.randn(*i.shape).astype(i.data.dtype) o.backward(o_grad) ref_grad = o_grad * mask * scale # Verify assert_allclose(i.g, orig_grad + ref_grad) # Check if accum option works i.g[...] = 1 o.g = o_grad o.parent.backward([i], [o], [False]) assert_allclose(i.g, ref_grad) # Check accum=False with NaN gradient i.g = np.float32('nan') o.parent.backward([i], [o], [False]) assert not np.any(np.isnan(i.g)) # Check if need_grad works i.g[...] = 0 i.need_grad = False o.backward(o_grad) assert np.all(i.g == 0)
def test_dropout_forward_backward(p, seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, function_tester rng = np.random.RandomState(seed) inputs = [ cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3))] # Ensure there is no zero. i = nn.Variable(inputs[0].shape, need_grad=True) i.d = inputs[0] # NNabla forward with nn.context_scope(ctx), nn.auto_forward(): o = F.dropout(i, p) scale = 1. / (1. - p) mask = o.d != 0 assert np.allclose(o.d, i.d * mask * scale) assert o.parent.name == func_name # NNabla backward orig_grad = rng.randn(*i.shape).astype(i.data.dtype) i.g[...] = orig_grad o_grad = rng.randn(*i.shape).astype(i.data.dtype) o.backward(o_grad) ref_grad = o_grad * mask * scale # Verify assert np.allclose(i.g, orig_grad + ref_grad) # Check if accum option works i.g[...] = 1 o.g = o_grad o.parent.backward([i], [o], [False]) assert np.allclose(i.g, ref_grad) # Check accum=False with NaN gradient i.g = np.float32('nan') o.parent.backward([i], [o], [False]) assert not np.any(np.isnan(i.g)) # Check if need_grad works i.g[...] = 0 i.need_grad = False o.backward(o_grad) assert np.all(i.g == 0)
def test_abs_forward_backward(seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, function_tester rng = np.random.RandomState(seed) inputs = [] for _ in range(2): inputs.append( cap_ignore_region( rng.randn( 2, 3, ).astype(np.float32) * 2, (-1e-3, 1e-3))) function_tester( rng, F.absolute_error, lambda x, y: np.abs(x - y), inputs, ctx=ctx, func_name=func_name, atol_b=1e-2) # NOTE if atol_b=1e-3, then a numerical error occurs.
def test_dropout_double_backward(p, seed, ctx, func_name): from nbla_test_utils import cap_ignore_region, backward_function_tester rng = np.random.RandomState(seed) inpd = cap_ignore_region( rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3)) # Ensure there is no zero. inp = nn.Variable.from_numpy_array(inpd).apply(need_grad=True) # ONLY test the double backward with nn.context_scope(ctx): dout = F.dropout(inp, p, seed) out = F.sigmoid(dout) # Check gradient w.r.t. dy only since no backward w.r.t. x grads = nn.grad([out], [inp]) grad = grads[0] grad.forward() grad.backward(1.0, clear_buffer=False) g_dy = grad.parent.inputs[1].g scale = 1. / (1. - p) mask = dout.d != 0 assert np.allclose(g_dy, mask * scale)