def test_fused_batch_normalization_forward_backward(seed, axis, decay_rate,
eps, nonlinearity,
output_stat, add, ctx,
func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = list(create_inputs(rng, axis, add))
axes = [axis]
batch_stat = True
function_tester(rng,
F.fused_batch_normalization,
ref_fused_batch_normalization,
inputs,
ref_grad=ref_grad_fused_batch_normalization,
func_args=[
axes, decay_rate, eps, batch_stat, nonlinearity,
output_stat
],
backward=[True, True, True, False, False, add],
ctx=ctx,
func_name=func_name,
dstep=1e-2,
atol_b=1e-2)
# Check if running mean and var works.
vinputs = []
for i in inputs:
if i is None:
vinputs.append(None)
continue
vinputs.append(nn.Variable.from_numpy_array(i, need_grad=True))
for i in range(5):
inputs[0] = rng.randn(*inputs[0].shape)
vinputs[0].d[...] = inputs[0]
ref_y = ref_fused_batch_normalization(
*(inputs +
[axes, decay_rate, eps, batch_stat, nonlinearity, output_stat]))
with nn.context_scope(ctx), nn.auto_forward():
y = F.fused_batch_normalization(*(
vinputs +
[axes, decay_rate, eps, batch_stat, nonlinearity, output_stat]
))
assert np.allclose(vinputs[3].d, inputs[3])
assert np.allclose(vinputs[4].d, inputs[4], atol=1e-3)
# Check if global stat mode works
batch_stat = False
if output_stat:
return
ref_y = ref_fused_batch_normalization(
*(inputs +
[axes, decay_rate, eps, batch_stat, nonlinearity, output_stat]))
with nn.context_scope(ctx), nn.auto_forward():
y = F.fused_batch_normalization(
*(vinputs +
[axes, decay_rate, eps, batch_stat, nonlinearity, output_stat]))
assert np.allclose(ref_y, y.d, atol=1e-6)
def test_fused_batch_normalization_forward_backward(
seed, axis, decay_rate, eps, nonlinearity, output_stat, add, ctx,
func_name, no_scale, no_bias, no_mean, no_variance):
import platform
if platform.system() == 'Windows' and len(ctx.backend) > 1:
pytest.skip("Currently not worked with CUDA/cuDNN on Windows platform."
) # TODO
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = list(create_inputs(rng, axis, add))
axes = [axis]
batch_stat = True
inputs = mask_inputs(inputs, no_scale, no_bias, no_mean, no_variance)
function_tester(rng,
F.fused_batch_normalization,
ref_fused_batch_normalization,
inputs,
ref_grad=ref_grad_fused_batch_normalization,
func_args=[
axes, decay_rate, eps, batch_stat, nonlinearity,
output_stat
],
backward=[True, True, True, False, False, add],
ctx=ctx,
func_name=func_name,
dstep=1e-2,
atol_b=1e-2)
# Check if running mean and var works.
if no_mean and no_variance:
return
vinputs = []
for i in inputs:
if i is None:
vinputs.append(None)
continue
vinputs.append(nn.Variable.from_numpy_array(i, need_grad=True))
for i in range(5):
inputs[0] = rng.randn(*inputs[0].shape)
vinputs[0].d[...] = inputs[0]
ref_y = ref_fused_batch_normalization(
*(inputs +
[axes, decay_rate, eps, batch_stat, nonlinearity, output_stat]))
with nn.context_scope(ctx), nn.auto_forward():
y = F.fused_batch_normalization(*(
vinputs +
[axes, decay_rate, eps, batch_stat, nonlinearity, output_stat]
))
assert_allclose(vinputs[3].d, inputs[3])
assert_allclose(vinputs[4].d, inputs[4], atol=1e-3)
# Check if global stat mode works
batch_stat = False
if output_stat:
return
ref_y = ref_fused_batch_normalization(
*(inputs +
[axes, decay_rate, eps, batch_stat, nonlinearity, output_stat]))
with nn.context_scope(ctx), nn.auto_forward():
y = F.fused_batch_normalization(
*(vinputs +
[axes, decay_rate, eps, batch_stat, nonlinearity, output_stat]))
assert_allclose(ref_y, y.d, atol=1e-6)