def test_random_shift_forward_backward(seed, inshape, shifts, border_mode,
constant_value, ctx, func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.random_shift(i, shifts, border_mode, constant_value, 0, seed)
result_shifts = (0, 0, 0)
max_correl = 0
for shift_amount in itertools.product(*map(
tuple,
map(lambda x: range(*x), [(-2, 3) for _ in range(len(inshape))]))):
r = scipy_shift(inputs[0],
shift_amount,
mode=border_mode,
cval=constant_value)
correl_and_p = pearsonr(o.d.flatten(), r.flatten())
if correl_and_p[0] > max_correl:
result_shifts = shift_amount
max_correl = correl_and_p[0]
ref = scipy_shift(inputs[0],
result_shifts,
mode=border_mode,
cval=constant_value)
if shifts is None:
shifts = (0, ) * len(inputs[0].shape)
for result, shift_range in zip(result_shifts, shifts):
assert abs(result) <= shift_range
assert_allclose(o.d, ref)
assert o.parent.name == func_name
# Skipping Backward check
g = np.random.randn(*i.shape)
i.g = g
o_grad = np.random.randn(*o.shape)
o.g = o_grad
o.parent.backward([i], [o])
ref_grad = i.g.copy() - g
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert_allclose(i.g, ref_grad, atol=1e-6)
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o_grad = rng.randn(*i.shape).astype(i.data.dtype)
o.backward(o_grad)
assert np.all(i.g == 0)
def test_ndarray_arithmetic_scalar_ops(seed, op, shape):
rng = np.random.RandomState(seed)
vx = nn.NdArray.from_numpy_array(rng.randn(*shape).astype(np.float32))
a = rng.randn()
if op == "**" and vx.size > 0:
vx.data += - vx.data.min() + 1.0
vz = eval("vx {0} a".format(op))
ref_z = eval("vx.data {0} a".format(op))
assert_allclose(ref_z, vz.data)
# Inplace test
vx_bak = vx
if op == '+':
vx += a
elif op == '-':
vx -= a
elif op == '*':
vx *= a
elif op == '/':
vx /= a
elif op == '**':
vx **= a
assert_allclose(vx.data, vz.data)
if op == "**":
# In-placing for `**` is obsoleted.
return
assert vx is vx_bak
def function_network_tester(rng,
func,
inputs,
func_args=[],
args_out=False,
have_scalar=False,
backward=None,
atol_b=1e-3,
dstep=1e-3):
'''
Automatic testing of backward of `func`
Args:
rng: random number generator
func: test function name
inputs: the inputs of func
func_args: the other func args of func
args_out: func_args whether contain output arguments, output arguments always set last
have_scalar: whether the operands contain scalars
backward: the attribute of nn.Variable
'''
inputs_ = inputs
if have_scalar:
if args_out:
inputs_ += func_args[:-1]
func_args = [func_args[-1]]
else:
inputs_ += func_args
func_args = []
if backward is None:
backward = [True for _ in inputs_]
variable_inputs = _create_variables(inputs_, backward)
# checking backward
if False in backward:
return
# NNabla backward
for v in variable_inputs:
v.g = _randn(rng, *v.shape)
y = func(*(variable_inputs + func_args))
if args_out:
y = func_args[-1]
nbla_grad, numerical_grad = compute_nnabla_and_numerical_grad(
inputs_,
variable_inputs,
y,
rng,
epsilon=dstep,
have_scalar=have_scalar)
assert_allclose(
nbla_grad,
numerical_grad,
atol=atol_b,
err_msg="{} backward w/o accumulation test fails.".format(func))
def test_reduce_scatter(seed, division, comm_nccl_opts):
if comm_nccl_opts is None:
pytest.skip(
"Communicator test is disabled. You can turn it on by an option `--test-communicator`.")
if len(comm_nccl_opts.devices) < 2:
pytest.skip(
"Communicator test is disabled. Use more than 1 gpus.")
comm = comm_nccl_opts.comm
device_id = int(comm_nccl_opts.device_id)
devices = comm_nccl_opts.devices
n_devices = len(comm_nccl_opts.devices)
# Variables
rng = np.random.RandomState(seed)
x_list = []
x_data_list = []
for i in devices:
x_data = rng.rand(3, 4)
x_data_list.append(x_data)
x = nn.Variable(x_data.shape)
x.d = x_data * (device_id + 1)
x_list.append(x)
# Reduce Scatter
x = nn.Variable((3, 4))
comm.reduce_scatter(
[x.data for x in x_list], x.data, division=division)
# Ref
refs = ref_reduce_scatter(x_data_list, n_devices, division)
# Check
assert_allclose(x.d, refs[device_id], rtol=1e-3, atol=1e-6)
def test_all_gather(seed, comm_nccl_opts):
if comm_nccl_opts is None:
pytest.skip(
"Communicator test is disabled. You can turn it on by an option `--test-communicator`."
)
if len(comm_nccl_opts.devices) < 2:
pytest.skip("Communicator test is disabled. Use more than 1 gpus.")
comm = comm_nccl_opts.comm
device_id = int(comm_nccl_opts.device_id)
n_devices = len(comm_nccl_opts.devices)
# Variables
rng = np.random.RandomState(seed)
x_data = rng.rand(3, 4)
x = nn.Variable(x_data.shape)
x.d = x_data * device_id
y_list = []
for i in range(n_devices):
y = nn.Variable(x_data.shape)
y_list.append(y)
# AllGahter
comm.all_gather(x.data, [y.data for y in y_list])
# Ref
refs = ref_all_gather(x_data, n_devices)
# Check
for y, ref in zip(y_list, refs):
assert_allclose(y.d, ref, rtol=1e-3, atol=1e-6)
def test_ndarray_arithmetic_ops2(seed, op, x_var, y_var, shape):
rng = np.random.RandomState(seed)
vx_data = rng.randn(*shape).astype(np.float32)
vy_data = rng.randn(*shape).astype(np.float32)
if op == "**" and vx_data.size > 0:
vx_data += -vx_data.min() + 1.0
if x_var:
vx = nn.Variable.from_numpy_array(vx_data)
else:
vx = nn.NdArray.from_numpy_array(vx_data)
if y_var:
vy = nn.Variable.from_numpy_array(vy_data)
else:
vy = nn.NdArray.from_numpy_array(vy_data)
vz = eval("vx {0} vy".format(op))
ref_z = eval("vx_data {0} vy_data".format(op))
assert_allclose(ref_z, vz.data)
if x_var:
return
# Inplace test
vx_bak = vx
exec_("vx {0}= vy".format(op))
assert_allclose(vx.data, vz.data)
assert vx is vx_bak
def test_confusion_matrix_forward(seed, ctx, axis, func_name):
ishape = [5, 6, 7]
rng = np.random.RandomState(seed)
l_shape = list(ishape)
l_shape[axis] = 1
n_class = ishape[axis]
inputs = [
rng.rand(5, 6, 7).astype(np.float32),
rng.randint(0, n_class, size=l_shape).astype(np.int)
]
ref = ref_confusion_matrix(inputs[0], inputs[1], axis)
x = nn.Variable(ishape)
l = nn.Variable(l_shape)
y = F.confusion_matrix(x, l, axis)
x.d = inputs[0]
l.d = inputs[1]
y.forward()
res = y.d
atol_f = 1e-6
assert_allclose(ref, res, atol=atol_f)
def test_nnp_graph_save_type(seed, tmpdir, need_file_object, file_type):
rng = np.random.RandomState(seed)
def unit(i, prefix):
c1 = PF.convolution(i, 4, (3, 3), pad=(1, 1), name=prefix + '-c1')
c2 = PF.convolution(F.relu(c1), 4,
(3, 3), pad=(1, 1), name=prefix + '-c2')
c = F.add2(c2, c1, inplace=True)
return c
x = nn.Variable([2, 3, 4, 4])
c1 = unit(x, 'c1')
c2 = unit(x, 'c2')
y = PF.affine(c2, 5, name='fc')
runtime_contents = {
'networks': [
{'name': 'graph',
'batch_size': 2,
'outputs': {'y': y},
'names': {'x': x}}],
}
nnp = get_nnp(runtime_contents, tmpdir, need_file_object, file_type)
graph = nnp.get_network('graph')
x2 = graph.inputs['x']
y2 = graph.outputs['y']
d = rng.randn(*x.shape).astype(np.float32)
x.d = d
x2.d = d
y.forward(clear_buffer=True)
y2.forward(clear_buffer=True)
assert_allclose(y.d, y2.d)
def inplace_function_test_helper(inputs,
func,
func_args=[],
func_kwargs={},
ctx=None,
func_name=None,
rng=None):
if rng is None:
rng = np.random.RandomState(313)
if ctx is None:
ctx = nn.Context()
with nn.context_scope(ctx):
a_s = [inp * 1.0 for inp in inputs]
y = func(*(a_s + list(func_args)), inplace=False, **func_kwargs)
l = F.sum(y)
a_s_i = [inp * 1.0 for inp in inputs]
y_i = func(*(a_s_i + list(func_args)), inplace=True, **func_kwargs)
l_i = F.sum(y_i)
data = [(randn(rng, *inp.shape), randn(rng, *inp.shape)) for inp in inputs]
for i in range(len(data)):
inputs[i].d = data[i][0]
inputs[i].g = data[i][1]
l.forward()
l.backward()
grads = [inp.g.copy() for inp in inputs]
for i in range(len(data)):
inputs[i].d = data[i][0]
inputs[i].g = data[i][1]
l_i.forward()
l_i.backward()
grads_i = [inp.g.copy() for inp in inputs]
for g, g_i in zip(grads, grads_i):
assert_allclose(g,
g_i,
err_msg="{} inplace test fails.".format(func_name))
def test_create_cache(test_data_csv_csv_20, test_data_csv_png_20,
input_file_fmt, cache_file_fmt, shuffle, normalize,
num_of_threads):
if input_file_fmt == 'csv':
csvfilename = test_data_csv_csv_20
else:
csvfilename = test_data_csv_png_20
nnabla_config.set('DATA_ITERATOR', 'cache_file_format', cache_file_fmt)
with create_temp_with_dir() as tmpdir:
cc = CreateCache(csvfilename,
shuffle=shuffle,
num_of_threads=num_of_threads)
cc.create(tmpdir, normalize=normalize)
# get cache data source and csv file data source
with closing(CacheDataSource(tmpdir)) as cache_source:
csv_source = CsvDataSource(csvfilename, normalize=normalize)
check_relative_csv_file_result(cache_file_fmt, csvfilename, tmpdir)
assert cache_source.size == csv_source.size
assert set(cache_source.variables) == set(csv_source.variables)
if shuffle:
with open(os.path.join(tmpdir, 'order.csv'), 'r') as f:
csv_source._order = [int(row[1]) for row in csv.reader(f)]
for _ in range(cache_source.size):
cache_data = associate_variables_and_data(cache_source)
csv_data = associate_variables_and_data(csv_source)
for v in cache_source.variables:
assert_allclose(cache_data[v], csv_data[v])
def value_tester(v_ref, v_act, rtol=1e-04, atol=1e-05):
from nbla_test_utils import ArrayDiffStats
v_ref.forward()
v_act.forward()
print(ArrayDiffStats(v_ref.d, v_act.d))
assert_allclose(v_ref.d, v_act.d, rtol=rtol, atol=atol)
def test_clip_by_value_forward(seed, shape, dtype):
def convert(value):
converter = dtype if dtype in (float,
np.array) else dtype.from_numpy_array
return converter(value)
rng = np.random.RandomState(seed)
x_data = rng.randn(*shape)
x = nn.Variable.from_numpy_array(x_data)
if dtype is float:
min_data = rng.randn()
max_data = rng.randn()
else:
min_data = rng.randn(*shape)
max_data = rng.randn(*shape)
min_ = convert(min_data)
max_ = convert(max_data)
if dtype is not np.array:
with nn.auto_forward(True):
y = F.clip_by_value(x, min_, max_)
y_ref = ref_clip_by_value(x_data, min_data, max_data)
if dtype in (nn.Variable, float):
assert_allclose(y.d, y_ref)
elif dtype is nn.NdArray:
assert_allclose(y.data, y_ref)
else:
with pytest.raises(TypeError):
y = F.clip_by_value(x, min_data, max_data)
def test_grad_grad_resnet(seed, ctx, auto_forward, inplace, shared):
nn.clear_parameters()
# Settings
nn.set_default_context(ctx)
nn.set_auto_forward(auto_forward)
b, c, h, w = 4, 3, 32, 32
n_cls = 10
rng = np.random.RandomState(seed)
# Network
x = nn.Variable.from_numpy_array(rng.randn(b, c, h,
w)).apply(need_grad=True)
y = SmallResNet(x, inplace=inplace, shared=shared)
# Grad of grad
dx = nn.grad([y], [x])
ddx = nn.grad([dx[0]], [x])
ddx[0].forward() if not auto_forward else None
# Backward of grad
x.grad.zero()
dx[0].forward() if not auto_forward else None
dx[0].backward()
# Check between results of var.backward and nn.grad
backend = ctx.backend[0].split(":")[0]
if backend == 'cuda':
pytest.skip(
'CUDA Convolution N-D is only supported in CUDNN extension')
assert_allclose(x.g, ddx[0].d, atol=1e-6)
def test_intermediate_outputs(clear_buffer, clear_no_need_grad):
rng = np.random.RandomState(311)
# unuse cached array to clear buffers immediately
nn.prefer_cached_array(False)
x = nn.Variable.from_numpy_array(rng.randn(2, 10))
h1 = x + 1
y1 = h1 + 1
h2 = x + 1
h2.persistent = True
y2 = h2 + 1
nn.forward_all([h1, y1],
clear_buffer=clear_buffer,
clear_no_need_grad=clear_no_need_grad)
nn.forward_all([h2, y2],
clear_buffer=clear_buffer,
clear_no_need_grad=clear_no_need_grad)
assert_allclose(h1.d, h2.d)
assert_allclose(y1.d, y2.d)
# revert perference (this is also done in conftest.py, but just in case)
nn.prefer_cached_array(True)
def test_random_erase_forward(ctx, func_name, seed, prob, area_ratios,
aspect_ratios, replacements, n, share, inplace,
base_axis, func_seed, channel_last):
if channel_last and func_name == "RandomErase":
pytest.skip("RandomErase with channel_last is only supported in CUDA.")
lb = replacements[0]
rng = np.random.RandomState(seed)
b, c, h, w = 4, 3, 32, 32
ishape = [b, h, w, c] if channel_last else [b, c, h, w]
x = nn.Variable.from_numpy_array(rng.rand(*ishape) + 1.0)
with nn.context_scope(ctx):
y0 = F.random_erase(x,
prob=prob,
area_ratios=area_ratios,
aspect_ratios=aspect_ratios,
replacements=replacements,
n=n,
share=share,
inplace=inplace,
base_axis=base_axis,
seed=func_seed,
channel_last=channel_last)
# Deterministic check
y0.forward()
if prob == 1.0:
assert np.any(y0.d >= lb)
# Random but with same seed check
if func_seed != -1:
with nn.context_scope(ctx):
y1 = F.random_erase(x,
prob=prob,
area_ratios=area_ratios,
aspect_ratios=aspect_ratios,
replacements=replacements,
n=n,
share=share,
inplace=inplace,
base_axis=base_axis,
seed=func_seed,
channel_last=channel_last)
y1.forward()
assert_allclose(y0.d, y1.d)
# Random but with different seed check
with nn.context_scope(ctx):
y2 = F.random_erase(x,
prob=prob,
area_ratios=area_ratios,
aspect_ratios=aspect_ratios,
replacements=replacements,
n=n,
share=share,
inplace=inplace,
base_axis=base_axis,
seed=func_seed + 2,
channel_last=channel_last)
y2.forward()
assert np.any(y0.d != y2.d)
def test_pow_scalar_inplace(val, seed, ctx, func_name):
inputs = [nn.Variable([2, 3, 4], need_grad=True)]
func_args = [val]
func_kwargs = {}
func = F.pow_scalar
rng = np.random.RandomState(seed)
# copied from `inplace_function_test_helper` function and modified
with nn.context_scope(ctx):
a_s = [inp * 1.0 for inp in inputs]
y = func(*(a_s + list(func_args)), inplace=False, **func_kwargs)
l = F.sum(y)
a_s_i = [inp * 1.0 for inp in inputs]
y_i = func(*(a_s_i + list(func_args)), inplace=True, **func_kwargs)
l_i = F.sum(y_i)
data = [((rng.randint(5, size=inp.shape).astype(np.float32) + 1.0) * 0.2,
rng.randn(*inp.shape)) for inp in inputs]
for i in range(len(data)):
inputs[i].d = data[i][0]
inputs[i].g = data[i][1]
l.forward()
l.backward()
grads = [inp.g.copy() for inp in inputs]
for i in range(len(data)):
inputs[i].d = data[i][0]
inputs[i].g = data[i][1]
l_i.forward()
l_i.backward()
grads_i = [inp.g.copy() for inp in inputs]
for g, g_i in zip(grads, grads_i):
assert_allclose(g, g_i)
def test_graph_more_than_2_outputs(seed, clear_buffer):
count = 0
def func_hook(f):
nonlocal count
if f.name == 'Split':
count += 1
nn.clear_parameters()
a = nn.Variable.from_numpy_array(np.ones((10, )))
b = nn.Variable.from_numpy_array(np.ones((10, )))
c = F.add2(a, b, inplace=True, outputs=[a.data])
y = F.split(c, axis=0)
nn.forward_all(y, function_pre_hook=func_hook)
assert count == 1
res = [x.d for x in y]
assert_allclose(res, [2.0] * 10)
a = nn.Variable.from_numpy_array(np.ones((10, )))
b = nn.Variable.from_numpy_array(np.ones((10, )))
c = F.add2(a, b, inplace=True, outputs=[a.data])
y = F.split(c, axis=0)
for yy in y:
yy.forward()
res = [x.d for x in y]
assert_allclose(res, [11.0] * 10)
def test_all_reduce(seed, inplace, division, comm_nccl_opts):
check_comm_nccl_opts(comm_nccl_opts)
comm = comm_nccl_opts.comm
device_id = int(comm_nccl_opts.device_id)
n_devices = len(comm_nccl_opts.devices)
# Variables
x_list = []
x_data_list = []
num_layers = 20
rng = np.random.RandomState(seed)
for l in range(num_layers):
x_data = rng.rand(3, 4)
x_data_list.append(x_data)
x = nn.Variable(x_data.shape)
x.d = x_data * (device_id + 1)
x_list.append(x)
# AllReduce
comm.all_reduce([x.data for x in x_list],
division=division,
inplace=inplace)
# Ref AllReduce
refs = ref_all_reduce(x_data_list, n_devices, division)
# Check
for x, ref in zip(x_list, refs):
assert_allclose(x.d, ref, rtol=1e-3, atol=1e-6)
def test_bcast(seed, src, inplace, comm_nccl_opts):
if comm_nccl_opts is None:
pytest.skip(
"Communicator test is disabled. You can turn it on by an option `--test-communicator`."
)
if len(comm_nccl_opts.devices) < 2:
pytest.skip("Communicator test is disabled. Use more than 1 gpus.")
comm = comm_nccl_opts.comm
device_id = int(comm_nccl_opts.device_id)
n_devices = len(comm_nccl_opts.devices)
# Variables
x_list = []
x_data_list = []
num_layers = 20
rng = np.random.RandomState(seed)
for l in range(num_layers):
x_data = rng.rand(3, 4)
x_data_list.append(x_data)
x = nn.Variable(x_data.shape)
x.d = x_data * (device_id + 1)
x_list.append(x)
# Bcast
comm.bcast([x.data for x in x_list], src, inplace=inplace)
# Ref
refs = ref_bcast(x_data_list, src)
# Check
for x, ref in zip(x_list, refs):
assert_allclose(x.d, ref, rtol=1e-3, atol=1e-6)
def test_add_n_active_inputs(n_inputs, n_active, input_shape, seed):
from nnabla.testing import assert_allclose
rng = np.random.RandomState(seed)
inputs = [rng.randn(*input_shape).astype('f4') for _ in range(n_inputs)]
active = np.random.permutation(n_inputs) < n_active
y = F.add_n(*[
nn.Variable.from_numpy_array(inp).apply(need_grad=True)
for inp in inputs
])
y.parent.set_active_input_mask(active)
y_ref = F.add_n(*[
nn.Variable.from_numpy_array(inp).apply(need_grad=True)
for (act, inp) in zip(active, inputs) if act
])
y.forward()
y_ref.forward()
assert_allclose(y.d, y_ref.d)
for inp in y.parent.inputs + y_ref.parent.inputs:
inp.g = 0
y.backward()
y_ref.backward()
active_inputs = [y.parent.inputs[i] for i, act in enumerate(active) if act]
for inp, ref in zip(active_inputs, y_ref.parent.inputs):
assert_allclose(inp.g, ref.g)
def test_variable_arithmetic_scalar_ops(seed, op):
rng = np.random.RandomState(seed)
vx = nn.Variable.from_numpy_array(rng.randn(2, 3, 4).astype(np.float32))
a = rng.randn()
if op == "**":
vx.d += - vx.d.min() + 1.0
with nn.auto_forward():
vz = eval("vx {0} a".format(op))
ref_z = eval("vx.d {0} a".format(op))
assert_allclose(ref_z, vz.d)
# Inplace test
with nn.auto_forward():
# Make function reference count of `vx` to 1.
vx = nn.functions.identity(vx)
vx_bak = vx
if op == "+":
vx += a
elif op == "-":
vx -= a
elif op == "*":
vx *= a
elif op == "/":
vx /= a
elif op == "**":
vx **= a
assert_allclose(vx.d, vz.d)
assert vx is not vx_bak
def test_variable_arithmetic_unary_ops(seed, op):
rng = np.random.RandomState(seed)
vx = nn.Variable.from_numpy_array(rng.randn(2, 3, 4).astype(np.float32))
with nn.auto_forward():
vz = eval("{0} vx".format(op))
ref_z = eval("{0} vx.d".format(op))
assert_allclose(ref_z, vz.d)
def test_graph_logreg(seed):
rng = np.random.RandomState(seed)
x = nn.Variable([2, 3, 4], need_grad=True)
w = nn.Variable([12, 5], need_grad=True)
b = nn.Variable([5], need_grad=True)
t = nn.Variable([2, 1])
x.d = rng.randn(*x.shape)
w.d = rng.randn(*w.shape)
b.d = rng.randn(*b.shape)
t.d = rng.randint(0, 5, size=t.shape)
nn.set_default_context(nn.Context())
# Forwardprop by definition
with nn.auto_forward():
z = F.affine(x, w, b, 1)
l = F.softmax_cross_entropy(z, t, 1)
L = F.mean(l)
# Backprop
# Diff should be initialized since they are always accumulated
x.g = 0
w.g = 0
b.g = 0
L.backward(clear_buffer=True)
x.g = rng.randn(*x.shape)
inputs = [x, w, b]
from nbla_test_utils import \
compute_analytical_and_numerical_grad_graph as grads
agrad, ngrad = grads(L, inputs, 1e-3)
assert_allclose(ngrad, agrad, atol=1e-2)
def test_apply_weight_standardization(rng, function, channel_axis, kwargs,
param_name):
eps = 1e-5
output_stat = False
"""
Graph overview of the weight standardization
inputs callback parent output
------- -------- ---------- --------
x --------------->|
w ------> WS ---->|----> function -----> y
b --------------->|
"""
x = nn.Variable.from_numpy_array(rng.randn(2, 8, 4, 4).astype(np.float32))
def ws_callback(w):
return F.weight_standardization(w,
channel_axis,
eps=eps,
output_stat=output_stat)
y = function(x, apply_w=ws_callback, **kwargs)
# check forward backward
y.forward()
y.backward()
w = nn.get_parameters()[param_name].d
w_standardized = y.parent.inputs[1].d
ref_w_standardized = ref_weight_standardization(w, channel_axis, eps,
output_stat)
assert_allclose(w_standardized, ref_w_standardized, atol=1e-02, rtol=1e-5)
def test_solver_zeroing():
xs = [nn.Variable([2, 3, 4], need_grad=True) for _ in range(3)]
s = S.Sgd(1)
s.set_parameters({str(i): x for i, x in enumerate(xs)})
for x in xs:
x.data.fill(1)
x.grad.zero()
s.weight_decay(1.0)
s.update()
for x in xs:
# Grad is not referenced since neither weight decay nor update is performed.
assert x.grad.zeroing
assert_allclose(x.d, 1)
for x in xs:
x.grad.fill(1)
s.weight_decay(0.1)
s.update()
for x in xs:
assert_allclose(x.d, 1 - (1 + 0.1))
def test_assign_forward_backward(seed, ctx, func_name):
rng = np.random.RandomState(seed)
dst = nn.Variable((2, 3, 4), need_grad=True)
src = nn.Variable((2, 3, 4), need_grad=True)
assign = F.assign(dst, src)
src.d = rng.rand(2, 3, 4)
assign.forward()
# destination variable should be equal to source variable
assert_allclose(dst.d, src.d)
# output variable of assign function should be equal to soure variable
assert_allclose(assign.d, src.d)
dummy = assign + rng.rand()
dst.grad.zero()
src.grad.zero()
dummy.forward()
dummy.backward()
# gradients at destination are identical to gradients at assign operation
assert not np.all(dst.g == np.zeros((2, 3, 4)))
assert np.all(dst.g == assign.g)
assert np.all(src.g == np.zeros((2, 3, 4)))
# check accum=False
assign.grad.zero()
dst.g = rng.rand(2, 3, 4)
f = assign.parent
f.forward([dst, src], [assign])
f.backward([dst, src], [assign], accum=[False])
assert np.all(dst.g == assign.g)
assert np.all(src.g == np.zeros((2, 3, 4)))
def test_top_n_error_forward(seed, axis, n, ctx, func_name):
ishape = [5, 6, 7]
rng = np.random.RandomState(seed)
l_shape = list(ishape)
l_shape[axis] = 1
n_class = ishape[axis]
inputs = [
rng.rand(5, 6, 7).astype(np.float32) * 0.9 + 0.05,
rng.randint(0, n_class, size=l_shape).astype(np.int)
]
ref = ref_top_n_error(inputs[0], inputs[1], axis, n)
x = nn.Variable(ishape)
l = nn.Variable(l_shape)
y = F.top_n_error(x, l, axis, n)
x.d = inputs[0]
l.d = inputs[1]
y.forward()
res = y.d
atol_f = 1e-6
assert_allclose(ref, res, atol=atol_f)
def test_imperative_i2_o1():
import nnabla.functions as F
x0 = nn.NdArray([2, 3, 4])
x1 = nn.NdArray([2, 1, 1])
x0.fill(3)
x1.fill(0.5)
y = F.mul2(x0, x1)
assert_allclose(y.data, 1.5)
def test_graph_model(model, seed):
np.random.seed(313)
rng = np.random.RandomState(seed)
x = nn.Variable([2, 3, 4, 4], need_grad=True)
t = nn.Variable([2, 1])
x.d = rng.randn(*x.shape)
t.d = rng.randint(0, 5, size=t.shape)
nn.set_default_context(nn.Context())
# Forwardprop by definition
nn.clear_parameters()
if model == "mlp":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 3)
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
elif model == "recurrent":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 8)
z2 = F.relu(z, inplace=True)
h = z2
for _ in range(2):
with nn.parameter_scope('fc2'):
h = PF.affine(h, 8)
h = F.relu(h, inplace=True)
with nn.parameter_scope('fc3'):
z3 = PF.affine(h, 5)
elif model == "convolution":
with nn.parameter_scope('conv1'):
z = PF.convolution(x, 3, (2, 2))
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
else:
raise ValueError()
l = F.softmax_cross_entropy(z3, t, 1)
L = F.mean(l)
# Forwardprop
L.forward(clear_no_need_grad=True)
# Backprop
# Diff should be initialized since they are always accumulated
x.grad.zero()
L.backward(clear_buffer=True)
x.g = rng.randn(*x.shape)
parameters = nn.get_parameters()
for param in parameters.values():
param.grad.zero()
inputs = [x] + list(parameters.values())
from nbla_test_utils import \
compute_analytical_and_numerical_grad_graph as grads
agrad, ngrad = grads(L, inputs, 1e-3)
assert_allclose(ngrad, agrad, atol=1.05e-2)
def test_shared_variable_on_same_function(seed):
rng = np.random.RandomState(313)
xd = rng.randn(2, 3)
x = nn.Variable.from_numpy_array(xd).apply(need_grad=True)
x.grad.zero()
y = x * x * x
y.forward()
y.backward()
assert_allclose(x.g, 3 * xd ** 2)
