def value_tester(v_ref, v_act, rtol=1e-04, atol=1e-05):
from nbla_test_utils import ArrayDiffStats
v_ref.forward()
v_act.forward()
print(ArrayDiffStats(v_ref.d, v_act.d))
assert_allclose(v_ref.d, v_act.d, rtol=rtol, atol=atol)
def value_tester(v_ref,
v_act,
rtol=1e-04,
atol=1e-05,
clear_no_need_grad=False):
from nbla_test_utils import ArrayDiffStats
v_ref.forward(clear_no_need_grad)
v_act.forward(clear_no_need_grad)
print(ArrayDiffStats(v_ref.d, v_act.d))
assert_allclose(v_ref.d, v_act.d, rtol=rtol, atol=atol)
def test_nnp_graph(seed):
rng = np.random.RandomState(seed)
def unit(i, prefix):
c1 = PF.convolution(i, 4, (3, 3), pad=(1, 1), name=prefix + '-c1')
c2 = PF.convolution(F.relu(c1),
4, (3, 3),
pad=(1, 1),
name=prefix + '-c2')
c = F.add2(c2, c1, inplace=True)
return c
x = nn.Variable([2, 3, 4, 4])
c1 = unit(x, 'c1')
c2 = unit(x, 'c2')
y = PF.affine(c2, 5, name='fc')
runtime_contents = {
'networks': [{
'name': 'graph',
'batch_size': 2,
'outputs': {
'y': y
},
'names': {
'x': x
}
}],
}
import tempfile
tmpdir = tempfile.mkdtemp()
import os
nnp_file = os.path.join(tmpdir, 'tmp.nnp')
try:
from nnabla.utils.save import save
save(nnp_file, runtime_contents)
from nnabla.utils import nnp_graph
nnp = nnp_graph.NnpLoader(nnp_file)
finally:
import shutil
shutil.rmtree(tmpdir)
graph = nnp.get_network('graph')
x2 = graph.inputs['x']
y2 = graph.outputs['y']
d = rng.randn(*x.shape).astype(np.float32)
x.d = d
x2.d = d
y.forward(clear_buffer=True)
y2.forward(clear_buffer=True)
from nbla_test_utils import ArrayDiffStats
assert np.allclose(y.d, y2.d), str(ArrayDiffStats(y.d, y2.d))
def test_grad_outputs(seed, ctx, auto_forward, type_grad_outputs):
from nbla_test_utils import ArrayDiffStats
# Settings
nn.set_default_context(ctx)
nn.set_auto_forward(auto_forward)
b, c, h, w = 4, 3, 32, 32
n_cls = 10
rng = np.random.RandomState(seed)
x = nn.Variable.from_numpy_array(rng.randn(b, c, h,
w)).apply(need_grad=True)
y = F.sigmoid(x)
# Grad outputs
if type_grad_outputs == int:
g = rng.randint(-10, 10)
elif type_grad_outputs == float:
g = rng.randn()
elif type_grad_outputs == np.ndarray:
g = rng.randn(*y.shape)
elif type_grad_outputs == nn.NdArray:
g = nn.NdArray.from_numpy_array(rng.randn(*y.shape))
# Zerograd, Forward, Backward on the forward graph
inputs = [x]
[inp.grad.fill(0) for inp in inputs]
if not auto_forward:
y.forward()
y.backward(g)
# Grad
inputs = [x]
outputs = [y]
grad_outputs = [g]
grads = nn.grad(outputs, inputs, grad_outputs)
if not auto_forward:
F.sink(*grads, one_input_grad=1).forward()
# Check between results of var.bacwkard and nn.grad
for inp, grad in zip(inputs, grads):
assert np.allclose(inp.g, grad.d,
atol=1e-6), str(ArrayDiffStats(inp.g, grad.d))
def test_multiple_objectives(seed, ctx, auto_forward):
from nbla_test_utils import ArrayDiffStats
# Settings
nn.set_default_context(ctx)
nn.set_auto_forward(auto_forward)
b, c, h, w = 4, 3, 32, 32
n_cls = 10
rng = np.random.RandomState(seed)
# Objecive0
x0 = nn.Variable.from_numpy_array(rng.randn(b, c, h,
w)).apply(need_grad=True)
y0 = F.sigmoid(x0)
# Objecive1
x1 = nn.Variable.from_numpy_array(rng.randn(b, c, h,
w)).apply(need_grad=True)
y1 = F.tanh(x1)
# Zerograd, Forward, Backward on the forward graph
g0 = nn.NdArray.from_numpy_array(rng.randn(*x0.shape))
g1 = nn.NdArray.from_numpy_array(rng.randn(*x1.shape))
z = y0 * nn.Variable(g0.shape).apply(data=g0) + y1 * \
nn.Variable(g1.shape).apply(data=g1)
inputs = [x0, x1]
[inp.grad.fill(0) for inp in inputs]
if not auto_forward:
z.forward()
z.backward()
# Grad
inputs = [x0, x1]
outputs = [y0, y1]
grad_outputs = [g0, g1]
grads = nn.grad(outputs, inputs, grad_outputs)
if not auto_forward:
F.sink(*grads, one_input_grad=1).forward()
# Check between results of var.bacwkard and nn.grad
for inp, grad in zip(inputs, grads):
assert np.allclose(inp.g, grad.d,
atol=1e-6), str(ArrayDiffStats(inp.g, grad.d))
def test_resnet_expansion(seed, ctx, auto_forward, flag_grad_outputs):
from nbla_test_utils import ArrayDiffStats
nn.clear_parameters()
# Settings
nn.set_default_context(ctx)
nn.set_auto_forward(auto_forward)
b, c, h, w = 4, 3, 32, 32
n_cls = 10
rng = np.random.RandomState(seed)
# Network
x = nn.Variable.from_numpy_array(rng.randn(b, c, h, w))
y = nn.Variable.from_numpy_array(rng.randint(0, n_cls, b).reshape(b, 1))
p = SmallResNet(x)
loss = F.mean(F.softmax_cross_entropy(p, y))
# Zerograd, Forward, Backward on the forward graph
inputs = nn.get_parameters().values()
[inp.grad.fill(0) for inp in inputs]
grad = nn.NdArray.from_numpy_array(np.asarray(
rng.randn())) if flag_grad_outputs else 1
if not auto_forward:
loss.forward()
loss.backward(grad)
# Grad
grad_outputs = grad if flag_grad_outputs else None
grads = nn.grad([loss], inputs, [grad_outputs])
if not auto_forward:
F.sink(*grads, one_input_grad=1).forward()
# Check between results of var.bacwkard and nn.grad
backend = ctx.backend[0].split(":")[0]
if backend == 'cuda':
pytest.skip(
'CUDA Convolution N-D is only supported in CUDNN extension')
for inp, grad in zip(inputs, grads):
assert np.allclose(inp.g, grad.d,
atol=1e-6), str(ArrayDiffStats(inp.g, grad.d))
def test_min_max_quantize_forward_backward(seed, x_shape, q_shape, decay,
x_min_max, ema, ste_fine_grained,
eps, quantize, ctx, func_name):
from nbla_test_utils import cap_ignore_region, \
function_tester
rng = np.random.RandomState(seed)
# Inputs
x = rng.randn(*x_shape)
qr_min = -0.5 * rng.rand(*q_shape)
qr_max = +0.5 * rng.rand(*q_shape)
ql_min = np.zeros(q_shape)
ql_max = np.ones(q_shape) * 255
inputs = [x, qr_min, qr_max, ql_min, ql_max]
func_args = [decay, x_min_max, ema, ste_fine_grained, eps, quantize]
# No quantization
if not quantize:
vinputs = [nn.Variable.from_numpy_array(xd) for xd in inputs]
v = vinputs[0]
with nn.context_scope(ctx):
o = F.min_max_quantize(*(vinputs + func_args))
np.allclose(o.d, v.d)
return
# x_min_max and ema
# function_tester does not work in this combination
# when comparing gradients between true_g = v.g - g_init (accum=True) and g (accum=False)
# since forward changes the exponential moving averages
if x_min_max and ema:
from nbla_test_utils import ArrayDiffStats
vinputs = [nn.Variable.from_numpy_array(xd) for xd in inputs]
vinputs[0].need_grad = True
with nn.context_scope(ctx):
y = F.min_max_quantize(*(vinputs + func_args))
# forward check
y.forward()
y_ref = ref_min_max_quantize(x, qr_min, qr_max, ql_min, ql_max, decay,
x_min_max, ema, ste_fine_grained, eps,
quantize)
assert np.allclose(y_ref, y.d, atol=1e-5), ArrayDiffStats(y_ref, y.d)
# backward check (accum=False)
xv = vinputs[0]
xv.grad.zero()
dy = rng.randn(*y.shape)
y.backward(dy)
gx_ref = ref_grad_min_max_quantize(x, qr_min, qr_max, ql_min, ql_max,
dy, decay, x_min_max, ema,
ste_fine_grained, eps, quantize)
ag = xv.g.copy()
assert np.allclose(gx_ref, ag.flatten(),
atol=1e-5), ArrayDiffStats(gx_ref, ag.flatten())
# backward check (accum=True)
y.backward(dy)
assert np.allclose(ag * 2.0, xv.g.copy(),
atol=1e-5), ArrayDiffStats(ag * 2.0, xv.g.copy())
return
# General tests
backward = [True, False, False, False, False, False] if x_min_max or ema \
else [True, True, True, False, False, False]
function_tester(rng,
F.min_max_quantize,
ref_min_max_quantize,
inputs,
func_args=func_args,
atol_b=1e-3,
backward=backward,
ctx=ctx,
func_name=func_name,
disable_half_test=True,
ref_grad=ref_grad_min_max_quantize)